diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c04ec6e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +# Conformance fixtures, evidence, and artifact manifests are content hashed, so a +# CRLF checkout would change their digests and break the gates. Keep every text file +# LF in the working tree on all platforms, and never translate the binary fixtures. +* text=auto eol=lf +*.parquet -text +*.parq -text +test/datasets/**/_common_metadata -text + +# Machine-generated and reproducible from their inputs, so collapse them by default +# in review. Regenerate rather than edit: the metadata types come from +# `julia thrift/generate.jl`, and the evidence records from the N6 producer harness. +src/metadata/parquet.jl linguist-generated=true +test/conformance/n6/evidence/*.jsonl linguist-generated=true + +# These frozen N6 artifacts include a pinned terminal blank line. +test/conformance/n6/oracles/parquet-java/check.sh whitespace=-blank-at-eof +test/conformance/n6/oracles/raw-java/.gitignore whitespace=-blank-at-eof diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bed26fa..8c5da4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,8 +13,8 @@ jobs: fail-fast: false matrix: version: - - '1.3' - - '1' # automatically expands to the latest stable 1.x release of Julia + - '1.10' + - '1' - 'nightly' os: - ubuntu-latest @@ -23,18 +23,67 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v5 - - uses: julia-actions/setup-julia@v2 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + repository: apache/parquet-testing + ref: 09f3cdbde45302f0f0c689c950e465e98a9df960 + path: test/parquet-testing + - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: julia-actions/cache@v2 - - uses: julia-actions/julia-buildpkg@v1 - - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2 + - uses: julia-actions/julia-buildpkg@e3eb439fad4f9aba7da2667e7510e4a46ebc46e1 # v1 + - name: Run N5 nested conformance + env: + PARQUET_TESTING_DIR: test/parquet-testing + run: julia --project=. test/conformance/n5/runtests.jl + - uses: julia-actions/julia-runtest@6e050c8013b833b1195105ff2fce9cd802f53271 # v1 env: JULIA_NUM_THREADS: 4 - - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v5 + - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38 # v1 + - uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: lcov.info + n6-static: + timeout-minutes: 20 + name: N6 static - Julia ${{ matrix.version }} - macOS arm64 + runs-on: macos-15 + strategy: + fail-fast: false + matrix: + version: + - '1.10.11' + - '1.12.6' + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2 + with: + version: ${{ matrix.version }} + arch: aarch64 + - name: Run N6 static preflight + env: + PARQUET_N6_GATE: '0' + run: julia --project=. --startup-file=no --history-file=no test/conformance/n6/runtests.jl + bounds: + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + repository: apache/parquet-testing + ref: 09f3cdbde45302f0f0c689c950e465e98a9df960 + path: test/parquet-testing + - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2 + with: + version: '1' + - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2 + - uses: julia-actions/julia-buildpkg@e3eb439fad4f9aba7da2667e7510e4a46ebc46e1 # v1 + - name: Run N5 nested conformance + env: + PARQUET_TESTING_DIR: test/parquet-testing + run: julia --project=. test/conformance/n5/runtests.jl + - run: julia --project --check-bounds=yes -e 'using Pkg; Pkg.test()' diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..5837226 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,31 @@ +name: Documentation +on: + push: + branches: [master] + tags: ['*'] + pull_request: +jobs: + build: + permissions: + actions: write + contents: write + pull-requests: read + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2 + with: + version: '1' + - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2 + - name: Install dependencies + shell: julia --color=yes --project=docs {0} + run: | + using Pkg + Pkg.develop(PackageSpec(path=pwd())) + Pkg.instantiate() + - name: Build and deploy + run: julia --color=yes --project=docs docs/make.jl + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.gitignore b/.gitignore index c4f35ef..5538798 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ parquet-compatibility/ julia-parquet-compatibility/ .vscode/settings.json +docs/build/ +# Resolved environments are never part of the branch. The N6 gate pins its own +# manifest under test/conformance/n6/julia/. +/Manifest.toml +/docs/Manifest.toml +/test/parquet-testing/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8a3796b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,17 @@ +# Parquet.jl contributor guide + +Read `HANDOFF.md` first when continuing the `rewrite/1.0` branch or preparing a release. +Read `docs/dev/architecture.md` before a change. Update `test/conformance/features.toml` only when the required evidence exists. + +- Keep the public surface small and namespaced. Do not add exports. +- Preserve unknown Thrift fields, enum values, and page kinds. +- Check a resource limit before every metadata-directed allocation. +- Use explicit `return` statements in functions. +- Use `T[]` for empty typed arrays. +- Keep functions small. Keep one empty line between functions. +- Wrap every `Threads.@spawn` task with `errormonitor`. +- Use `@atomic` fields instead of `Atomic{T}`. +- Add a focused regression test for every fix. +- Confirm written files with at least one independent Parquet implementation. + +Do not copy code from Parquet3.jl. That repository has no license. Parquet2.jl is MIT, but copied work needs attribution and a license notice. diff --git a/Artifacts.toml b/Artifacts.toml deleted file mode 100644 index 4f0bd93..0000000 --- a/Artifacts.toml +++ /dev/null @@ -1,15 +0,0 @@ -[julia_parcompat] -git-tree-sha1 = "066eb71b5392a8edb036e6a66a92d9b94a9e7eed" -lazy = true - - [[julia_parcompat.download]] - sha256 = "943ac718383a9bb1144e0edc88ec1c42f071e00d40728af7b18a213fd4da5d1c" - url = "https://github.com/JuliaIO/parquet-compatibility/archive/3f7586f1b7f2a0c6b048791fb5f97c0b3df52e39.tar.gz" - -[parcompat] -git-tree-sha1 = "1e993c153d3df6b2039ea5df61aeea2cb5213753" -lazy = true - - [[parcompat.download]] - sha256 = "895f382e65e4684335d6cbd2d682172d0923660f5525e0682112361305f05b64" - url = "https://github.com/Parquet/parquet-compatibility/archive/2b47eac447c7a4a88247651a4065984db7b247ff.tar.gz" diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 0000000..1d59dde --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,198 @@ +# Parquet.jl `rewrite/1.0` handoff + +Last updated: 2026-08-27 + +Read this file before continuing the rewrite. It separates implemented code, +verified evidence, planned scope, and release authority. + +## Branch state + +- Local and remote branch: `rewrite/1.0` +- Baseline: `95a3037aa1e643370f04c4d2393c2e925a7d3115` +- Baseline package: Parquet.jl 0.8.6 +- Rewrite package version: `1.0.0-DEV` +- Stable format target: Apache Parquet 2.13.0 +- Pinned format source: `c47e2a66e88943fc46fde1b028a9432f14fdf5c0` +- Pinned corpus: `09f3cdbde45302f0f0c689c950e465e98a9df960` +- Current disposition: preproduction +- Pull request: none +- Release authority: none + +The branch preserves the registered package UUID and Git history. It removes the old +PAR2 runtime and replaces it with a new pure Julia implementation. + +## Authority boundary + +`test/conformance/n6/manifest.toml` is authoritative for the exact N6 evidence scope. +It intentionally contains: + +```toml +status = "preproduction" +supported_platforms = ["macos-15-arm64"] +publication_authorized = false +oracle_lock_authorized = false +``` + +Do not call this branch release-ready. Do not publish an oracle lock, tag a release, +or change either authorization flag until the remaining provenance and platform gates +are complete and the user gives explicit approval. + +The requested Claude Fable 5 Max review did not run. Claude Code reported an +insufficient credit balance. No Claude agreement or implementation approval exists. +Independent Codex agents performed the confirmed adversarial reviews. The final N6 +review found no P0, P1, or P2 issue in its reviewed preproduction scope. + +## What changed + +The rewrite added these working layers: + +- bounded source ownership and footer framing; +- a pure Julia Thrift Compact Protocol runtime; +- generated Parquet 2.13.0 metadata types; +- physical and logical schema validation; +- Dremel levels and nested vectors; +- PLAIN, RLE, dictionary, delta, and BYTE_STREAM_SPLIT encodings; +- Data Page V1 and V2 framing with CRC checks; +- UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, and LZ4 paths; +- scalar logical types and bounded JSON, BSON, and decimal validation; +- recursive nested reading and writing slices; +- statistics, producer-trust, page-index, and offset-index slices; +- a Tables.jl facade and namespaced writer API; +- resource-limit, mutation, corpus, interoperability, and evidence tests; +- Documenter documentation and CI workflows. + +The package has no exports. The actual public declaration is in `src/Parquet.jl`. +The current public names are `BSONValue`, `Decimal`, `File`, `Interval`, `JSONValue`, +`Limits`, `LogicalColumn`, `Table`, `Timestamp`, `close!`, and `write`. + +The public API and complete feature descriptions in `docs/dev/roadmap.md` are target +design. They are not proof that the corresponding module exists. For example, +`Dataset`, scan pushdown, bloom filters, encryption, Variant, and geospatial modules +are still target work. + +## Repository map + +| Area | Start here | Main verification | +| --- | --- | --- | +| Contribution rules | `AGENTS.md`, `SKILL.md` | Review every changed file against both | +| Architecture and gates | `docs/dev/architecture.md`, `docs/dev/roadmap.md` | `test/conformance/features.toml` | +| Source ownership | `src/source.jl`, `src/footer.jl` | `test/source.jl`, `test/footer.jl`, `test/limits.jl` | +| Metadata | `src/thrift.jl`, `src/metadata/parquet.jl` | `test/thrift.jl`, `test/metadata.jl`, `test/generator.jl` | +| Schema and nesting | `src/schema.jl`, `src/nested_schema.jl`, `src/dremel.jl` | `test/nested_schema.jl`, `test/nested_reader.jl`, `test/nested_table.jl` | +| Encodings | `src/plain.jl`, `src/rle.jl`, `src/delta.jl`, `src/bss.jl` | Matching files under `test/` | +| Pages and codecs | `src/page.jl`, `src/codecs.jl`, `src/checksum.jl` | `test/page.jl`, `test/codecs.jl`, `test/checksum.jl` | +| Logical values | `src/logical*.jl` | `test/logical*.jl` | +| Statistics and indexes | `src/statistics.jl`, `src/page_index.jl` | `test/statistics.jl`, `test/write_statistics.jl`, `test/write_offset_index.jl` | +| Writer | `src/write*.jl` | `test/write*.jl` | +| Tables facade | `src/table.jl`, `src/nested_table.jl` | `test/table.jl`, `test/nested_table.jl` | +| N5 conformance | `test/conformance/n5/` | `test/conformance/n5/runtests.jl` | +| N6 evidence gate | `test/conformance/n6/README.md`, `test/conformance/n6/manifest.toml` | `test/conformance/n6/runtests.jl` | +| User documentation | `README.md`, `docs/src/` | `docs/make.jl` | + +## Verified evidence + +The final preproduction verification on 2026-08-24 recorded: + +- full package suites passed on Julia 1.10.11 and 1.12.6; +- N6 external gate passed 5,117 of 5,117 checks; +- the independent model passed 578 of 578 checks; +- N6 static lanes passed 5,063 of 5,063 checks on both Julia versions; +- the Python harness passed 23 of 23 checks; +- normalizer tests passed 26 of 26 checks; +- all 69 pinned artifact hashes passed; +- strict docs, doctests, and public API checks passed; +- the final source composite hash was + `cdb21788f1c7c4d29e567681ecd851fb6dcdbe115bb29243e006cecb35080c05`. + +Before the branch push on 2026-08-27, `Pkg.test("Parquet")` passed again on +Julia 1.10.11 and 1.12.6 from temporary resolved environments. These reruns included +the local N5 and N6 harness tests. Corpus-only tests had the expected skips described +below. + +Exact N6 file identities at that gate: + +- `test/conformance/n6/runtests.jl`: + `b0708ac70a942093a631e849a2442169fd64376908b5e2b979055cb51fb7a3eb` +- `test/conformance/n6/manifest.toml`: + `f6761f7c13a80688e64651918aa9db1c46452a8f1c99a2c597a859ff4a82b2bf` +- `test/conformance/n6/artifacts.sha256`: + `dd5bf9b64b843597eabbec70b611bc7c39341ccce847b66279e25aca96fce4e6` + +The ordinary corpus tests skip corpus-only cases when `test/parquet-testing` is +absent. The exact N6 run used the authenticated external corpus. Keep that distinction +in all reports. + +## Common validation + +From the repository root: + +```sh +julia +1.10.11 --project=. --startup-file=no --history-file=no test/runtests.jl +julia +1.12.6 --project=. --startup-file=no --history-file=no test/runtests.jl +PARQUET_N6_GATE=0 julia +1.12.6 --project=. --startup-file=no --history-file=no test/conformance/n6/runtests.jl +julia +1.12.6 --project=docs --startup-file=no --history-file=no docs/make.jl +git diff --check +``` + +The exact N6 external gate needs authenticated source trees, runtime archives, wheels, +the JDK, the raw Java download cache, and Docker. Its complete environment contract is +in `test/conformance/n6/README.md`. Do not replace it with ambient Python, Java, Julia, +or package installations. + +The CI workflow runs package and N5 tests on Linux, macOS, and Windows. It also has +macOS ARM64 N6 static lanes. A branch-only push does not run the current push workflow, +because push events are limited to `master`. A pull request would run CI, but no pull +request was requested for this handoff. + +## Known remaining work + +1. Reconcile implementation, tests, docs, and `test/conformance/features.toml`. + The ledger remains conservative: stages 1 through 4 are `in_progress`, and stages + 5 through 8 are `planned`. Promote a row only after its full evidence contract + passes. +2. Complete the target-only modules: bloom filters, residual-safe scan pushdown, + modular encryption, Variant, geospatial support, and datasets. +3. Prove PyArrow and DuckDB source-to-wheel provenance. Their official wheel bytes and + runtime behavior are verified, but both source entries remain `planned` in the N6 + manifest. +4. Expand the exact gate beyond macOS 15 ARM64. Run clean Linux, Windows, other macOS, + Julia nightly, bounds, reverse-dependency, PkgEval, performance, and allocation + qualification. +5. Rebuild and review all current user-facing support statements. `README.md` and some + roadmap current-state paragraphs understate later nested, statistics, and index + slices. Treat tests and frozen evidence as facts until the text is reconciled. +6. Keep publication and oracle locking disabled until every release gate is complete. + +## Out of scope + +These are decided, not pending. Do not reopen them without a concrete user need. + +- LZO. Every available implementation is GPL-2, so an MIT core cannot depend on one. + A file that uses it reports an unsupported feature. +- INT96. Deprecated in the format and not supported in either direction. +- Writing the deprecated LZ4 codec and the deprecated BIT_PACKED encoding. Both remain + readable, because files in the wild use them; new files use LZ4_RAW and RLE. + +Complete coverage of the format is explicitly not a goal. No mainstream implementation +has it, and the practical target is interoperability with the implementations people +actually use. + +## Safe continuation order + +1. Start from a clean clone of `origin/rewrite/1.0`. +2. Read `AGENTS.md`, this handoff, `docs/dev/architecture.md`, and the relevant plan. +3. Confirm the branch tip and N6 hashes before changing source. +4. Select one conservative feature-ledger row. +5. Add valid, invalid, resource-limit, and independent interoperability evidence. +6. Run focused tests, both supported Julia suites, static N6, docs, and the relevant + external oracle. +7. Update a feature status only when the complete evidence contract is satisfied. +8. Report package, CI, review, provenance, and release readiness as separate gates. + +## Local checkout note + +This working directory can contain ignored N5/N6 build caches, downloaded toolchains, +and compiled oracle output under `test/conformance/`. They are not part of the branch. +A fresh clone reconstructs only the checked-in sources, fixtures, manifests, and +normalized evidence. The committed branch must not contain a root `Manifest.toml`, +`docs/Manifest.toml`, `docs/build`, Python bytecode, private keys, or access tokens. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..13d4150 --- /dev/null +++ b/NOTICE @@ -0,0 +1,7 @@ +Parquet.jl includes the Apache Parquet format IDL at `thrift/parquet.thrift`. +The IDL is licensed under the Apache License, Version 2.0. +It is pinned from apache/parquet-format commit +`a9f9c3a52bd1d6309038f4d2d3a308978b55c377` (format 2.13.0). + +Apache Parquet +Copyright The Apache Software Foundation diff --git a/Project.toml b/Project.toml index 4540075..616dace 100644 --- a/Project.toml +++ b/Project.toml @@ -3,42 +3,39 @@ uuid = "626c502c-15b0-58ad-a749-f091afb673ae" keywords = ["parquet", "julia", "columnar-storage"] license = "MIT" desc = "Julia implementation of parquet columnar file format reader and writer" -version = "0.8.6" +version = "1.0.0-DEV" [deps] -CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" -CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" -CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2" +ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" +ChunkCodecLibBrotli = "653b0ff7-85b5-4442-93c1-dcc330d3ec7d" +ChunkCodecLibLz4 = "7e9cc85e-5614-42a3-ad86-b78f920b38a5" +ChunkCodecLibSnappy = "eac87354-86d5-4a5b-ab5f-a6ee56b239b3" +ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" +ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" +CRC32 = "b4567568-9dcc-467e-9b62-c342d3a501d3" DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Decimals = "abce61dc-4473-55a0-ba07-351d65e31d42" -LittleEndianBase128 = "1724a1d5-ab78-548d-94b3-135c294f96cf" -Missings = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" -SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -Snappy = "59d4ed8c-697a-5b28-a4c7-fe95c22820f9" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -Thrift = "8d9c9c80-f77e-5080-9541-c6f69d204e22" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -CategoricalArrays = "0.6,0.7,0.8,0.9,0.10, 1" -CodecZlib = "0.5,0.6,0.7,0.8" -CodecZstd = "0.6,0.7,0.8" +ChunkCodecCore = "1" +ChunkCodecLibBrotli = "1" +ChunkCodecLibLz4 = "1" +ChunkCodecLibSnappy = "1" +ChunkCodecLibZlib = "1" +ChunkCodecLibZstd = "1" +CRC32 = "1" DataAPI = "1" -Decimals = "0.4" -LittleEndianBase128 = "0.3" -Missings = "0.3,0.4,1" -SentinelArrays = "1" -Snappy = "0.3, 0.4" -Tables = "1.6" -Thrift = "0.8" -julia = "1.3" +Tables = "1.12" +julia = "1.10" [extras] -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [targets] -test = ["Artifacts", "LazyArtifacts", "Random", "Test"] +test = ["Random", "SHA", "Test", "TOML"] diff --git a/README.md b/README.md index 4821358..2ca6788 100644 --- a/README.md +++ b/README.md @@ -1,233 +1,75 @@ -# Parquet - -[![CI](https://github.com/JuliaIO/Parquet.jl/actions/workflows/ci.yaml/badge.svg)](https://github.com/JuliaIO/Parquet.jl/actions/workflows/ci.yaml) -[![codecov](https://codecov.io/gh/JuliaIO/Parquet.jl/graph/badge.svg?token=qchPEYSd5Q)](https://codecov.io/gh/JuliaIO/Parquet.jl) - - -See also alternatives: [Parquet2.jl](https://gitlab.com/ExpandingMan/Parquet2.jl); We suggest also considering [DuckDB.jl](https://github.com/duckdb/duckdb) which is backed by a mature and well-maintained C++ library and has query support (see also [QuackIO.jl](https://github.com/JuliaAPlavin/QuackIO.jl) for a simple convenience wrapper of this). - -## Reader - -A [parquet file](https://en.wikipedia.org/wiki/Apache_Parquet) or dataset can be loaded using the `read_parquet` function. A parquet dataset is a directory with multiple parquet files, each of which is a partition belonging to the dataset. - -`read_parquet(path; kwargs...)` returns a `Parquet.Table` or `Parquet.Dataset`, which is the table contained in the parquet file or dataset in an Tables.jl compatible format. - -Options: -- `rows`: The row range to iterate through, all rows by default. Applicable only when reading a single file. -- `filter`: Filter function to apply while loading only a subset of partitions from a dataset. The path to the partition is provided as a parameter. -- `batchsize`: Maximum number of rows to read in each batch (default: row count of first row group). Applied only when reading a single file, and to each file when reading a dataset. -- `use_threads`: Whether to use threads while reading the file; applicable only for Julia v1.3 and later and switched on by default if julia processes is started with multiple threads. -- `column_generator`: Function to generate a partitioned column when not found in the partitioned table. Parameters provided to the function: table, column index, length of column to generate. Default implementation determines column values from the table path. - -The returned object is a Tables.jl compatible Table and can be converted to other forms, e.g. a `DataFrames.DataFrame` via - -```julia -using Parquet, DataFrames -df = DataFrame(read_parquet(path)) -``` - -Partitions in a parquet file or dataset can also be iterated over using an iterator returned by the `Tables.partitions` method. - -```julia -using Parquet, DataFrames -for partition in Tables.partitions(read_parquet(path)) - df = DataFrame(partition) - ... -end -``` - -### Lower Level Reader - -Load a [parquet file](https://en.wikipedia.org/wiki/Apache_Parquet). Only metadata is read initially, data is loaded in chunks on demand. (Note: [ParquetFiles.jl](https://github.com/queryverse/ParquetFiles.jl) also provides load support for Parquet files under the FileIO.jl package.) - -`Parquet.File` represents a Parquet file at `path` open for reading. - -``` -Parquet.File(path) => Parquet.File -``` - -`Parquet.File` keeps a handle to the open file and the file metadata and also holds a weakly referenced cache of page data read. If the parquet file references other files in its metadata, they will be opened as and when required for reading and closed when they are not needed anymore. - -The `close` method closes the reader, releases open files and makes cached internal data structures available for GC. A `Parquet.File` instance must not be used once closed. - -```julia -julia> using Parquet - -julia> filename = "customer.impala.parquet"; - -julia> parquetfile = Parquet.File(filename) -Parquet file: customer.impala.parquet - version: 1 - nrows: 150000 - created by: impala version 1.2-INTERNAL (build a462ec42e550c75fccbff98c720f37f3ee9d55a3) - cached: 0 column chunks -``` - -Examine the schema. - -```julia -julia> nrows(parquetfile) -150000 - -julia> ncols(parquetfile) -8 - -julia> colnames(parquetfile) -8-element Array{Array{String,1},1}: - ["c_custkey"] - ["c_name"] - ["c_address"] - ["c_nationkey"] - ["c_phone"] - ["c_acctbal"] - ["c_mktsegment"] - ["c_comment"] - -julia> schema(parquetfile) -Schema: - schema { - optional INT64 c_custkey - optional BYTE_ARRAY c_name - optional BYTE_ARRAY c_address - optional INT32 c_nationkey - optional BYTE_ARRAY c_phone - optional DOUBLE c_acctbal - optional BYTE_ARRAY c_mktsegment - optional BYTE_ARRAY c_comment - } -``` - -The reader performs logical type conversions automatically for String (from byte arrays), decimals (from fixed length byte arrays) and DateTime (from Int96). It depends on the converted type being populated correctly in the file metadata to detect such conversions. To take care of files where such metadata is not populated, an optional `map_logical_types` argument can be provided while opening the parquet file. The `map_logical_types` value must map column names to a tuple of return type and converter functon. Return types of String and DateTime are supported as of now, and default implementations for them are included in the package. +# Parquet.jl + +Parquet.jl is being rebuilt as a complete pure-Julia implementation of the Apache Parquet format. + +The `rewrite/1.0` branch is development work. It is not ready for data use. The +current foundation contains bounded byte sources, footer framing, a pure-Julia +Compact Protocol runtime, generated 2.13.0 metadata types, schema-tree and level +validation, and bounded encoding kernels. The current vertical slice reads flat Data +Page V1 and V2 files with PLAIN, dictionary, delta, Boolean RLE, and BYTE_STREAM_SPLIT +values. It writes every stable nondeprecated flat value encoding through +`Parquet.Table` and `Parquet.write`, including a name-based per-column policy. It +supports UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, and LZ4_RAW pages, plus deprecated +Hadoop and raw-block LZ4 input. The Stage 4 scalar layer reads STRING, ENUM, UUID, JSON, +BSON, DATE, TIME, TIMESTAMP, INTEGER, DECIMAL, FLOAT16, INTERVAL, and UNKNOWN values. It +writes these annotations from unambiguous Julia values or tagged package values, and a +`Parquet.Table` rewrite preserves the source scalar schema. The first nested slice +decodes three-level optional lists of supported primitive values and writes canonical +`optional LIST` columns. It preserves null lists, empty lists, null +elements, and present elements. Recursive and legacy nested forms, structs, maps, +indexes, encryption, Variant, and geospatial data are still in progress. LZO and INT96 +are not supported and are not planned: every LZO implementation is GPL-2, and INT96 is +deprecated in the format. +`NTuple{N,UInt8}` writer columns map to FIXED_LEN_BYTE_ARRAY, and `Parquet.Table` +preserves their runtime width across later writes. ```julia -julia> mapping = Dict(["column_name"] => (String, Parquet.logical_string)); - -julia> parquetfile = Parquet.File("filename"; map_logical_types=mapping); +Parquet.write("output.parquet", table; codec=:zstd, dictionary=true, + encoding=(id=:delta_binary_packed, measurement=:byte_stream_split), + pageversion=:v2, statistics=true) ``` -The reader will interpret logical types based on the `map_logical_types` provided. The following logical type mapping methods are available in the Parquet package. - -- `logical_timestamp(v; offset=Dates.Second(0))`: Applicable for timestamps that are `INT96` values. This converts the data read as `Int128` types to `DateTime` types. -- `logical_string(v)`: Applicable for strings that are `BYTE_ARRAY` values. Without this, they are represented in a `Vector{UInt8}` type. With this they are converted to `String` types. -- `logical_decimal(v, precision, scale; use_float=true)`: Applicable for reading decimals from `FIXED_LEN_BYTE_ARRAY`, `INT64`, or `INT32` values. This converts the data read as those types to `Integer`, `Float64` or `Decimal` of the given precision and scale, depending on the options provided. - -Variants of these methods or custom methods can also be applied by caller. - -### BatchedColumnsCursor - -Create cursor to iterate over batches of column values. Each iteration returns a named tuple of column names with batch of column values. Files with nested schemas can not be read with this cursor. - -```julia -BatchedColumnsCursor(parquetfile::Parquet.File; kwargs...) -``` +An `encoding` Symbol or string applies to every column. A `Pair`, `NamedTuple`, or +dictionary supplies exact column-name overrides. Unlisted columns remain PLAIN, or +use adaptive dictionary encoding when `dictionary=true`. Use `:dictionary` for an +adaptive dictionary override on one column. -Cursor options: -- `rows`: the row range to iterate through, all rows by default. -- `batchsize`: maximum number of rows to read in each batch (default: row count of first row group). -- `reusebuffer`: boolean to indicate whether to reuse the buffers with every iteration; if each iteration processes the batch and does not need to refer to the same data buffer again, then setting this to `true` reduces GC pressure and can help significantly while processing large files. -- `use_threads`: whether to use threads while reading the file; applicable only for Julia v1.3 and later and switched on by default if julia processes is started with multiple threads. +The writer emits row-group statistics by default. Set `statistics=false` to omit +them. `Parquet.Limits(max_statistics_value_bytes=4096)` limits each raw minimum or +maximum value before the writer copies it into file metadata. Counts and column +order remain available when a bound is too large to emit. -Example: +Use `Parquet.LogicalColumn` when the Julia element type does not contain the complete +Parquet schema. It can select ENUM, a TIME or TIMESTAMP unit and UTC flag, or DECIMAL +precision and scale. It also supplies a schema for empty and all-null columns. ```julia -julia> typemap = Dict(["c_name"]=>(String,Parquet.logical_string), ["c_address"]=>(String,Parquet.logical_string)); +using Dates -julia> parquetfile = Parquet.File("customer.impala.parquet"; map_logical_types=typemap); - -julia> cc = BatchedColumnsCursor(parquetfile) -Batched Columns Cursor on customer.impala.parquet - rows: 1:150000 - batches: 1 - cols: c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_mktsegment, c_comment - -julia> batchvals, state = iterate(cc); - -julia> propertynames(batchvals) -(:c_custkey, :c_name, :c_address, :c_nationkey, :c_phone, :c_acctbal, :c_mktsegment, :c_comment) - -julia> length(batchvals.c_name) -150000 - -julia> batchvals.c_name[1:5] -5-element Array{Union{Missing, String},1}: - "Customer#000000001" - "Customer#000000002" - "Customer#000000003" - "Customer#000000004" - "Customer#000000005" +values = Union{Missing,Dates.Time}[missing, Dates.Time(12)] +time = Parquet.LogicalColumn(values, :time; unit=:micros, adjusted=false) +Parquet.write("time.parquet", (; time)) ``` -### RecordCursor +`Parquet.JSONValue` validates [RFC 8259](https://www.rfc-editor.org/info/rfc8259/) +syntax. `Parquet.BSONValue` validates the +[BSON 1.1 document grammar](https://bsonspec.org/spec.html). Both validators are +bounded and do not build an object tree. Binary DECIMAL conversion has its own +`Limits.max_decimal_bytes` resource bound. -Create cursor to iterate over records. In parallel mode, multiple remote cursors can be created and iterated on in parallel. +## Target -```julia -RecordCursor(parquetfile::Parquet.File; kwargs...) -``` +- Apache Parquet format 2.13.0 is the stable contract. +- Post-2.13 features such as ALP remain experimental until they are released. +- The reader will accept all stable encodings and codecs, including deprecated input. +- The writer will emit all nondeprecated stable encodings and codecs. +- The core will not depend on Arrow.jl or a native Thrift compiler. +- The public API will remain small and namespaced under `Parquet`. -Cursor options: -- `rows`: the row range to iterate through, all rows by default. -- `colnames`: the column names to retrieve; all by default +Support is complete only after valid read coverage, valid write coverage when applicable, malformed-input coverage, and confirmation by an independent Parquet implementation. -Example: - -```julia -julia> typemap = Dict(["c_name"]=>(String,Parquet.logical_string), ["c_address"]=>(String,Parquet.logical_string)); +See [the development architecture](docs/dev/architecture.md) and [the machine-readable feature ledger](test/conformance/features.toml). -julia> parquetfile = Parquet.File("customer.impala.parquet"; map_logical_types=typemap); +## License -julia> rc = RecordCursor(parquetfile) -Record Cursor on customer.impala.parquet - rows: 1:150000 - cols: c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_mktsegment, c_comment - -julia> records = collect(rc); - -julia> length(records) -150000 - -julia> first_record = first(records); - -julia> isa(first_record, NamedTuple) -true - -julia> propertynames(first_record) -(:c_custkey, :c_name, :c_address, :c_nationkey, :c_phone, :c_acctbal, :c_mktsegment, :c_comment) - -julia> first_record.c_custkey -1 - -julia> first_record.c_name -"Customer#000000001" - -julia> first_record.c_address -"IVhzIApeRb ot,c,E" -``` - -## Writer - -You can write any Tables.jl column-accessible table that contains columns of these types and their union with `Missing`: `Int32`, `Int64`, `String`, `Bool`, `Float32`, `Float64`. - -However, `CategoricalArray`s are not yet supported. Furthermore, these types are not yet supported: `Int96`, `Int128`, `Date`, and `DateTime`. - -### Writer Example - -```julia -tbl = ( - int32 = Int32.(1:1000), - int64 = Int64.(1:1000), - float32 = Float32.(1:1000), - float64 = Float64.(1:1000), - bool = rand(Bool, 1000), - string = [randstring(8) for i in 1:1000], - int32m = rand([missing, 1:100...], 1000), - int64m = rand([missing, 1:100...], 1000), - float32m = rand([missing, Float32.(1:100)...], 1000), - float64m = rand([missing, Float64.(1:100)...], 1000), - boolm = rand([missing, true, false], 1000), - stringm = rand([missing, "abc", "def", "ghi"], 1000) -) - -file = tempname()*".parquet" -write_parquet(file, tbl) -``` +Parquet.jl is available under the MIT license. diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..d976b28 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,10 @@ +# Working on Parquet.jl + +1. Read `docs/dev/architecture.md` and `test/conformance/features.toml`. +2. Identify the exact Parquet 2.13.0 clause and corpus fixtures for the change. +3. Add valid, invalid, and resource-limit tests. +4. Implement the smallest complete layer change. +5. Run the focused tests and the full Julia test suite. +6. Run the relevant external oracle before changing a feature status to complete. + +Never infer format support from a successful package load or a local-only round trip. diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 0000000..91d0fbc --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,8 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Parquet = "626c502c-15b0-58ad-a749-f091afb673ae" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[compat] +Documenter = "1.17" +Tables = "1.12" diff --git a/docs/dev/architecture.md b/docs/dev/architecture.md new file mode 100644 index 0000000..88cdf1b --- /dev/null +++ b/docs/dev/architecture.md @@ -0,0 +1,67 @@ +# Parquet.jl 1.0 architecture + +Parquet.jl 1.0 is a new implementation. It keeps the registered package identity and repository history. It does not keep the old runtime. + +The detailed support policy and acceptance gates are in [`roadmap.md`](roadmap.md). + +## Contract + +The stable contract is Apache Parquet format 2.13.0 at peeled source commit `c47e2a66e88943fc46fde1b028a9432f14fdf5c0`. The release tag object is not used as a commit pin. Post-tag items such as ALP remain experimental. The reader accepts the stable encodings and codecs it supports, including deprecated encodings. The writer emits nondeprecated stable encodings and codecs. LZO and INT96 are out of scope in both directions and are not a release gate: the GPL-2 LibLZO package cannot be part of this MIT core, and INT96 is deprecated in the format. Complete format coverage is not claimed. + +Every support claim needs four facts: + +1. A valid file can be read. +2. A valid file can be written when the feature is writable. +3. Invalid input fails safely under explicit resource limits. +4. An independent implementation confirms interoperability. + +## Layers + +- `source.jl`: bounded byte-range input and explicit ownership. +- `thrift.jl`: pure Julia Compact Protocol with unknown-field preservation. +- `metadata/`: immutable code generated from the pinned Parquet IDL. +- `schema.jl`: physical, logical, and Dremel level interpretation. +- `plain.jl`, `rle.jl`, `delta.jl`, `bss.jl`, `dictionary.jl`: typed encoding kernels. +- `codecs.jl`: exact-size decompression charged to a resource budget first. +- `page.jl`: page framing, standard CRC32, and Data Page V1/V2. +- `logical.jl`, `logical_temporal.jl`, `logical_binary.jl`, + `logical_decimal.jl`, `logical_json.jl`, and `logical_bson.jl`: validated + physical-to-logical scalar conversion and bounded embedded-document syntax. +- `dremel.jl` and `vectors.jl`: nested assembly and owned or borrowed vectors. +- `read.jl` and `scan.jl`: bounded parallel decode and residual-safe pushdown. +- `write.jl`, `write_logical.jl`, and `logical_column.jl`: schema-aware row-group, + page, and logical-value production. +- `dataset.jl`: partition discovery and schema unification. +- `crypto.jl`: Parquet encryption framing and key-provider interfaces. +- `variant.jl` and `geo.jl`: complete Variant and geospatial modules. + +The core does not depend on Arrow.jl. Arrow interoperation belongs in an extension. The package has no exports. Users call the narrow API through `Parquet`. + +## Correctness rules + +- Preserve unknown enum values, unknown union members, field 32767, and unknown pages. +- Trust page headers. Do not treat footer encoding or offset lists as complete. +- Use standard CRC32. CRC32C is a different polynomial. +- Decode selected columns and columns referenced by filters. +- Prune only with a proof. Keep the exact filter as residual work. +- Treat absent statistics as unknown, not zero or empty. +- Apply the Parquet float, NaN, signed-zero, binary truncation, and `created_by` trust rules. +- Start nested V2 pages at repetition level zero. +- Preserve an existing table's scalar schema when a Julia runtime type cannot carry + Parquet annotation details such as a time unit, UTC flag, or declared precision. +- Bound all metadata-directed allocation before it occurs. +- Never follow a `file_path` outside the opened dataset root. + +## Delivery stages + +1. Repository, IDL, feature ledger, CI, and architecture. +2. Compact Protocol, generated metadata, and mutation tests. +3. Primitive PLAIN/RLE reader and writer vertical slice. +4. All encodings and compression codecs. +5. Logical types, Dremel nesting, and nested writer. +6. Statistics, indexes, bloom filters, and `Tables.Scan`. +7. Modular encryption. +8. Variant and geospatial data. +9. Dataset behavior, performance, fuzzing, docs, and release gates. + +Green unit tests do not make a release. Parquet 1.0 also requires the pinned parquet-testing corpus, PyArrow and DuckDB bidirectional checks, nightly parquet-java and arrow-rs checks, fuzzing, allocation checks, docs, licensing, and registered `Tables.Scan` support. diff --git a/docs/dev/n5-plan.md b/docs/dev/n5-plan.md new file mode 100644 index 0000000..1c2fb09 --- /dev/null +++ b/docs/dev/n5-plan.md @@ -0,0 +1,640 @@ +# N5 nested conformance and hardening plan + +This document refines N5 of the nested implementation plan. It is an +implementation agreement, not a statement of current support. Production code +does not change until the independent review of this plan reaches agreement. + +## Binding inputs + +- Apache Parquet format 2.13.0 at commit + `c47e2a66e88943fc46fde1b028a9432f14fdf5c0`. +- Apache `parquet-testing` at commit + `09f3cdbde45302f0f0c689c950e465e98a9df960`. +- Julia 1.10 and the current stable Julia release. +- Parquet Java tag `apache-parquet-1.17.1`, peeled commit + `78a8d3230eb4769db93de5f2f2e18363c04cae81`. +- Arrow Rust tag `59.2.0`, commit + `782e5a685501a9db6cc8e9a3b7cbff894940c47a`. + +The Java and Rust projects are test oracles. They do not become package +dependencies. Their wrappers, lock files, source revisions, and artifact hashes +are checked in or verified before execution. + +## Scope + +N5 closes the Stage 4 nested-data gate. It adds generated wire-format evidence, +hostile-input coverage, exact resource boundaries, and durable Java and Rust +oracles for legacy nested layouts. + +N5 does not implement ColumnIndex contents, statistics, bloom filters, predicate +pushdown, encryption, Variant, geospatial values, Dataset, or release work. It +does not change the public API or add exports. + +## Baseline and open evidence + +The current suite has schema-plan tests for all five specified LIST compatibility +rules. It uses six internal rule IDs because rule 4 has two exact name forms: +`array` and `_tuple`. Synthetic `LeafStream` tests cover values for each +rule. Full serialized corpus coverage exists for rules 1, 3, and 5, but not rule +2 or either rule-4 name. + +MAP schema-plan tests cover modern and legacy outer annotations, standalone +`MAP_KEY_VALUE`, positional child names, optional-key compatibility, omitted +values, and marked entry groups. Full serialized tests cover several standard, +Impala, optional-key, omitted-value, and package-written forms. Standalone outer +`MAP_KEY_VALUE`, arbitrary child names, and direct legacy LIST-of-MAP do not yet +have serialized independent fixtures. Prior PyArrow and DuckDB checks cover +canonical output, but they are temporary evidence and not the required durable +Java and Rust gate. + +The pinned corpus supplies canonical nested data, repeated unannotated data, +Impala list and map forms, omitted map values, optional map keys, old nested +lists, and a large-page limit fixture. The fixed tests are strong but do not yet +provide all required evidence. + +The missing evidence is: + +- An independent recursive shredder and assembler model. +- Serialized files for LIST rule 2, both rule-4 names, standalone outer + `MAP_KEY_VALUE`, arbitrary MAP child names, and direct legacy LIST-of-MAP. +- Deterministic generated schemas and values across recursive combinations. +- Mutations between count, validation, and emission passes. +- Exact success and one-byte-or-one-element-short resource boundaries. +- Durable Parquet Java and Arrow Rust generation and verification jobs. + +### Binding nested corpus + +CI verifies the corpus commit and these SHA-256 values before tests run. A wrong +hash or missing file fails the gate; it does not produce a skip. + +```text +5988ab91b6cb7efa7bf6a77f789b40929212280519be6c9daad56e01d5ceb218 list_columns.parquet +e64a64ff130c8dff64a6bc41480c51c87918d5e63bc75167b58524aa0fa01496 null_list.parquet +44f29191b5fa8cfe0ab848495bd8ef89344ac0d8f87b3dff12e267631e2b5c03 datapage_v2.snappy.parquet +065b336c65885ab9dfd97cf85ce39a45488ed12d0183917db0a11621b0711e3b old_list_structure.parquet +2cb2cc0564486a28550429a8b6d0907bbb41e138546797bc91a4ebd850edd5a5 nested_lists.snappy.parquet +db1a493003a7dcd2011bf89e460fed007903fcdeb58f53df29387b4e908e2a6d nested_maps.snappy.parquet +fcd6152058b8b8259a516105da5919b23cb8ccfc42258de0fe20e3107f8ef809 repeated_primitive_no_list.parquet +97d35acb9721e40fc0f66fba916a442c4a1cf77a35992dc891ad0cdcc5a24cfb repeated_no_annotation.parquet +de9102a599d852be3af1d2af5d3498d8e019c329096a6f2d260f55ae2d6ed0ae nullable.impala.parquet +e7927cde24c083e42a3d4b37ac962d34381f71c2d252169b627dd8459a5880e3 nonnullable.impala.parquet +5c4fc6c13fe7308acb2fd317a3bd59e5b9c9c206c005e863ae0a1abdbbf5e2ea map_no_value.parquet +5591dde252b46bc238a88e9c02e35780c5eb086e2677df105aaa91ff1fde8fba incorrect_map_schema.parquet +48427178bfef9e6edd9018f2ef7b084077c00057234a780271a8220ca53b33da nested_structs.rust.parquet +1ce6839f093ebc0699b1e2769ed04036bab40405bacbb5dacdd376dd94c13451 large_string_map.brotli.parquet +``` + +The first thirteen fixtures are the materializable rewrite gate. This includes an +exact schema-bearing rewrite of `datapage_v2.snappy.parquet`. +`large_string_map.brotli.parquet` instead gets exact footer and schema checks, +exact small-value-leaf checks, and required rejection of the 1 GiB key page before +large decompression or value materialization under default limits. Its complete +high-memory read is an optional manual gate. + +## Test ownership and independence + +The reference model is test-only. It owns its schema AST, semantic values, +level calculation, and normalization. It must not call the production nested +planner, count pass, emit pass, zipper, or assembler when it computes expected +results. + +The test fixture emitter may reuse the package's Compact Thrift scalar runtime, +page framing, compression, and primitive value encoders. Its schema layout and +repetition and definition streams must come from the independent model. + +A production writer-to-production reader round trip is integration evidence only. +It does not prove either implementation independently. Writer conformance compares +the model streams directly with the production pre-encode leaf streams. A small +test-owned page and hybrid-level decoder then extracts repetition, definition, and +dense-value streams from serialized bytes without calling the production page, +level, column, or table reader. Reader conformance compares production assembly +with the independently generated streams and independent inverse assembler. + +Maps use an ordered vector of pairs as their authoritative physical semantic +form. This preserves duplicate keys. A separate logical projection verifies the +specified last-value-wins behavior. JSON objects are not an authoritative map +manifest because they cannot preserve duplicate keys. + +Generated tests use a small checked-in SplitMix64 implementation. They do not +use Julia's default random-number stream as a cross-version contract. Every +failure prints the seed, case ID, schema AST, semantic rows, and leaf streams. +The canonical manifest encoder uses only explicitly ordered arrays, fixed-width +integer text, and length-prefixed UTF-8. It must not use `hash`, `Dict` or `Set` +iteration order, `show`, or Julia Serialization. + +## N5-A: deterministic compatibility model and goldens + +Add a test-local model for primitive leaves, structs, lists, and maps. It must +support required, optional, and repeated nodes and produce: + +- A flattened physical schema. +- Maximum repetition and definition levels for every leaf. +- Semantic rows. +- Exact repetition, definition, and dense-value streams. +- Package vector trees. +- Normalized semantic output. +- V1 and V2 serialized fixtures. + +### LIST matrix + +One mandatory golden set covers: + +1. A repeated primitive. +2. A repeated group with multiple fields. +3. A repeated group with one repeated child. +4. A repeated one-field group named `array`. +5. A repeated one-field group named `_tuple`. +6. A rule-5 wrapper with a required child. +7. A rule-5 wrapper with an optional child. +8. A direct legacy LIST-of-MAP case. + +Each applicable case contains null, empty, null-element, and present states. +Rule precedence is explicit: + +- A multi-field `array` uses rule 2. +- A one-field `array` whose child is repeated uses rule 3. +- `Array`, `ARRAY`, and case-mismatched tuple names use rule 5. +- The exact `array` and `_tuple` names use rule 4. +- Modern-only, converted-only, and matching dual LIST annotations have the same + semantic values. +- An unknown modern annotation blocks legacy LIST fallback. + +The following semantic rows and leaf streams are binding. `null` is a null outer +list, and tuple syntax describes a structured element. + +| Case | Semantic rows | Repetition | Definition | Dense values | +|---|---|---|---|---| +| Rule 1 and required-child rule 5 | `null`, `[]`, `[10]`, `[20,30]` | `[0,0,0,0,1]` | `[0,1,2,2,2]` | `[10,20,30]` | +| Rule 2 field `x` | `null`, `[]`, `[(1,null)]`, `[(2,20),(3,30)]` | `[0,0,0,0,1]` | `[0,1,2,2,2]` | `[1,2,3]` | +| Rule 2 field `y` | same rows | `[0,0,0,0,1]` | `[0,1,2,3,3]` | `[20,30]` | +| Rule 3 | `null`, `[]`, `[[]]`, `[[1,2],[],[3]]` | `[0,0,0,0,2,1,1]` | `[0,1,2,3,3,2,3]` | `[1,2,3]` | +| Rule 4 `array` | `null`, `[]`, `[(null)]`, `[(4),(null)]` | `[0,0,0,0,1]` | `[0,1,2,3,2]` | `[4]` | +| Rule 4 `_tuple` | `null`, `[]`, `[(7)]`, `[(null),(8)]` | `[0,0,0,0,1]` | `[0,1,3,2,3]` | `[7,8]` | +| Paired optional-child rule 5 | `null`, `[]`, `[null]`, `[4,null]` | `[0,0,0,0,1]` | `[0,1,2,3,2]` | `[4]` | +| Extended optional-child rule 5 | `null`, `[]`, `[null]`, `[5,null,6]` | `[0,0,0,0,1,1]` | `[0,1,2,3,2,3]` | `[5,6]` | +| Direct LIST-of-MAP key | `null`, `[]`, `[{}]`, `[{1=>10,1=>20},{},{2=>30}]` | `[0,0,0,0,2,1,1]` | `[0,1,2,3,3,2,3]` | `[1,1,2]` | +| Direct LIST-of-MAP value | same rows | `[0,0,0,0,2,1,1]` | `[0,1,2,3,3,2,3]` | `[10,20,30]` | + +The binding rule-3 schema is the Parquet 2.13 compatibility example: the +repeated inner group itself carries a LIST annotation and contains one repeated +primitive. An otherwise identical unannotated group has an ordinary group type +whose field is an unannotated repeated list. It is a separate serialized control, +not the binding `LIST>` golden. An external reader may report a looser +unwrapped interpretation only as diagnostic evidence. + +The golden manifest stores these arrays plus exact schema elements, leaf paths, +and file hashes. Rule 4 and rule 5 intentionally include identical physical +streams under different wrapper names. Their different semantic shapes prove +that the name rule is applied. + +### MAP grammar matrix + +Generate the complete accepted grammar and its rejected neighbors. A normal MAP +outer group is required or optional and has exactly one repeated group child. +The accepted outer marker is a modern logical MAP, a legacy converted MAP, or a +standalone converted `MAP_KEY_VALUE`. A modern MAP annotation takes precedence +over any converted annotation, including a conflict. An unknown modern annotation +blocks converted fallback and leaves an ordinary group. Any other winning modern +annotation makes the field not a MAP and is classified under that annotation; +it rejects only when that annotation is itself illegal on the group or topology. +Matching and conflicting dual annotations are explicit cases. + +The repeated entry is a group with one or two children. Its converted annotation +is absent or `MAP_KEY_VALUE`. Standalone outer `MAP_KEY_VALUE` plus an inner +`MAP_KEY_VALUE` marker is accepted as a tolerant read layout but is never emitted. +An absent entry annotation and an unknown future annotation are treated as +unmarked. A known non-MAP logical or converted annotation rejects the MAP layout. + +The first entry child is the key. It is required or compatibility-optional, but +never repeated. The second child is the value. It is required, optional, or +absent, but never repeated. Entry, key, and value names do not control +interpretation. The matrix serializes every accepted combination of outer marker, +outer repetition, entry marker, key repetition, value presence, and canonical or +arbitrary names. It adds one minimized rejected case for every neighboring +topology or annotation rule. + +This truth table is binding: + +| Location | Metadata or shape | Result | +|---|---|---| +| Outer | logical MAP, any converted value | Accept as modern MAP; the modern annotation wins. | +| Outer | no logical annotation, converted MAP | Accept as legacy MAP. | +| Outer | no logical annotation, converted `MAP_KEY_VALUE` | Accept as standalone legacy MAP alias. | +| Outer | unknown modern annotation plus converted MAP or `MAP_KEY_VALUE` | Do not apply converted fallback; compile as an ordinary future group. | +| Outer | no logical or converted annotation | Not a MAP; compile as an ordinary struct. | +| Outer | no logical annotation plus unknown converted annotation | Not a MAP; compile as an ordinary future-compatible group and preserve metadata. | +| Outer | modern LIST plus converted MAP or `MAP_KEY_VALUE` | Not a MAP; classify as LIST because modern metadata wins. | +| Outer | modern VARIANT, EMPTY, or other valid future group metadata plus converted MAP | Not a MAP; classify as the winning ordinary/future group. | +| Outer | primitive-only modern annotation on a group | Reject because the winning annotation is illegal on a group. | +| Outer | no logical annotation plus converted LIST | Not a MAP; classify as LIST. | +| Outer | no logical annotation plus primitive-only known converted annotation | Reject because the winning annotation is illegal on a group. | +| Outer | required or optional repetition | Accept. | +| Outer | repeated repetition outside a parent LIST compatibility rule | Reject. | +| Outer | repeated repetition owned by a parent LIST rule 3 | Accept as the tolerant direct LIST-of-MAP form. | +| Entry | repeated group with one or two children | Accept. | +| Entry | primitive, non-repeated, zero-child, or three-or-more-child form | Reject. | +| Entry | no annotation | Accept as unmarked. | +| Entry | converted `MAP_KEY_VALUE` | Accept as marked. | +| Entry | an explicitly empty logical annotation | Accept as unmarked. | +| Entry | unknown future logical annotation, including one paired with converted `MAP_KEY_VALUE` | Accept as unmarked; modern unknown metadata blocks the converted marker and is preserved. | +| Entry | unknown converted annotation with no logical annotation | Accept as unmarked and preserve metadata. | +| Entry | any known logical annotation, including logical MAP, LIST, or VARIANT | Reject at the entry position. | +| Entry | any known converted annotation other than `MAP_KEY_VALUE` | Reject at the entry position. | +| Key | first child, required | Accept as specified MAP. | +| Key | first child, optional | Accept only as the named existing-file compatibility exception; every physical key must still be present. | +| Key | repeated or absent | Reject. | +| Value | second child, required or optional | Accept. | +| Value | absent | Accept as key-only MAP. | +| Value | repeated or followed by another child | Reject. | +| Names | canonical or arbitrary | Accept by position; names do not select roles. | + +Keys and values may be primitive or recursive group shapes. Their internal schema +must independently satisfy the normal struct, LIST, MAP, repetition, and leaf +rules. Matching dual outer MAP annotations, conflicting converted metadata under +a modern MAP, and unknown-modern blocking each have serialized controls. + +Every accepted layout gets a serialized value case. Every rejected neighboring +layout gets a schema failure case. Required fixed cases are: + +- Standard optional MAP with optional values and duplicate keys. +- Positional MAP fields with arbitrary names. +- Standalone outer `MAP_KEY_VALUE`. +- Key-only MAP with an omitted value field. +- Compatibility-optional keys with every physical key present. +- A mutation with an actual null key, which must fail eagerly. +- A direct legacy LIST-of-MAP. + +Accepted value cases include null and empty maps when the outer repetition +allows them, required and null values when allowed, duplicate keys, and arbitrary +field names. Actual null keys always fail. The physical ordered-pair result and +the logical last-value-wins projection are checked separately. + +The following MAP rows and streams are binding: + +- Standard optional MAP rows are `null`, `{}`, `{a=>null}`, + `{a=>1,a=>2,b=>3}`, and `{c=>4}`. Repetition is + `[0,0,0,0,1,1,0]`. Key definitions are `[0,1,2,2,2,2,2]` with + dense keys `["a","a","a","b","c"]`. Value definitions are + `[0,1,2,3,3,3,3]` with dense values `[1,2,3,4]`. +- A required key-only MAP has rows `{}`, `{k1}`, and `{k2,k2}`. Repetition + is `[0,0,0,1]`. Key definitions are `[0,1,1,1]` with dense keys + `["k1","k2","k2"]`. +- A compatibility-optional-key MAP has rows `null`, `{}`, `{a=>1}`, and + `{b=>2,c=>3}`. Repetition is `[0,0,0,0,1]`. Key definitions are + `[0,1,3,3,3]` with dense keys `["a","b","c"]`. Required-value + definitions are `[0,1,2,2,2]` with dense values `[1,2,3]`. Changing one + present key definition from `3` to `2` creates an actual null key and must + fail eagerly. + +Direct repeated MAP inside a compatibility LIST is a tolerant package extension. +The binding integer-key Julia case owns its exact ordered pairs and streams. A +separately identified Java fixture with the same LIST/MAP topology and required +UTF-8 keys must produce an inferred Avro `LIST` schema and exact normalized +rows. This authorizes the disputed nesting only. The Java Avro type system cannot +authorize the integer-key domain, which remains low-level physical evidence. + +N5-A gate: + +- Every mandatory LIST and MAP layout has an exact V1 and V2 file test. +- Independent streams match production pre-encode writer leaf streams. +- The test-owned wire decoder recovers those streams from serialized pages. +- Production reader assembly matches the independent inverse assembler. +- Julia reads every fixture to the expected semantic value. +- Schema-bearing rewrites preserve the exact source schema and raw unknown + fields. +- Repeated writes of package-owned fixtures are byte deterministic. + +## N5-B: deterministic recursive properties + +Generate 256 bounded valid cases with seed `0x4e355f4c49535435`. + +- Depth is 1 through 6. +- Width is 1 through 4. +- Row count is 0 through 24. +- Collection length is 0 through 5. +- One case has at most 64 AST nodes, 16 physical leaves, 2,048 level entries, + 2,048 dense values, and 256 KiB of uncompressed primitive payload. +- Struct, every LIST rule, every accepted MAP form, and primitive leaves can + occur recursively where the format allows them. +- Every optional container emits null, empty, and present states when row count + permits. +- Every optional element or value emits null and present states when row count + permits. +- Map batches include duplicate keys. + +The generator records coverage counters. The test fails if any required node, +rule, repetition, null state, empty state, duplicate-key state, V1/V2 page form, +or row boundary is absent. + +Candidate case IDs are the fixed integers 0 through 4,095. Each candidate derives +its stream from the binding seed and its ID. A candidate that exceeds a hard cap +is rejected without partial fixture output. The first 256 accepted IDs are the +suite. Failure to obtain exactly 256 cases is an error. This fixed attempt schedule +prevents runtime timing or rejection order from changing the suite. + +For each case: + +1. Generate the semantic rows and schema AST. +2. Shred them with the independent model. +3. Assemble those streams with the independent inverse model. +4. Require `rows == independent_assemble(independent_shred(rows))`. +5. Compare exact production pre-encode writer leaf streams. +6. Serialize and independently decode the page streams. +7. Read with the production reader and require its normalized table to equal the + independent assembly. +8. Rewrite a schema-bearing table and compare exact source schema and values. +9. Repeat the write and compare bytes for deterministic package-owned output. + +All cases run with alternating V1 and V2 pages on Julia 1.10 and current stable. +Mandatory goldens run with both page versions. A stable 32-case subset runs with +all six supported writer codecs. + +N5-B gate: + +- All 256 cases pass on both Julia versions. +- Coverage counters prove every required category ran. +- The six-codec subset passes exact Julia reads and external secondary checks. +- Failure output is sufficient to replay one case without the generator. +- The ordered canonical manifest for all 256 cases has one checked SHA-256 digest. + Julia 1.10 and current stable must both reproduce it exactly. + +## N5-C: mutation and resource hardening + +Add hostile package-vector and Tables sources that mutate one fact between +inspection, count, validation, and emission. Cover: + +- Length and axes. +- Child identity, order, count, and names. +- Validity bits, ranks, offsets, and terminal offsets. +- List and map entry counts. +- Map key presence and pair order. +- Payload sizes and logical conversions. +- Schema topology, annotations, repetitions, row counts, and leaf paths. +- Page entry counts, dense counts, fixed widths, and row-group counts. +- Dictionary ordering and advertised dictionary, data, and index offsets. + +Add serialized mutations for invalid LIST and MAP child counts and repetitions, +zero-field repeated wrappers, annotation conflicts, null optional keys, +continuations above the parent repetition level, sibling boundary and occurrence +disagreement, dense underflow and overflow, split rows, and schema-bearing +rewrite topology changes. + +Page-boundary tests follow the wire contract. A V2 page cannot split a logical +row. A V1 page must start at a row boundary when an OffsetIndex advertises it; +without an OffsetIndex, a legacy V1 continuation is accepted. Correctly framed +and checksum-valid INDEX_PAGE and unknown page frames before data are accepted +controls only in a chunk without a dictionary or when they occur after its +dictionary. Truncation, checksum failure, overlap, type/subheader mismatch, or a +false advertised offset rejects. A dictionary must be the first physical frame, +and every positive explicit dictionary offset remains exact. + +The error policy is: + +- Detectable structural mutation or inconsistent source objects throw + `ArgumentError`. Same-shape, same-size value changes between passes are outside + the promised mutation-detection contract. +- Malformed files throw `FormatError`. +- Resource exhaustion throws `LimitError` for the exact resource. +- No raw `BoundsError`, `OverflowError`, `InexactError`, `MethodError`, or + other package-originated implementation error escapes validation. +- Failed private operations restore caller-visible buffers and operation-owned + live-byte budgets to their entry state. + +Mutation detection applies at observable package access boundaries. A successful +user callback must not mutate an unrelated object that the callback did not +return. A transient change that is fully restored before any dependent package +access is outside the detection contract. For an arbitrary `AbstractDict`, the +writer first materializes its complete ordered `Pair` sequence. It snapshots +directly exposed package vectors and package-owned view backings as they become +observable. After the terminal iterator callback returns `nothing`, every such +base or dynamically discovered source must still match its snapshot before the +writer traverses dependent values. A persistent mismatch throws `ArgumentError`. +The writer does not use `length(dict)` as dictionary authority. + +Exceptions deliberately thrown by user `Tables`, vector, IO, or sink methods may +propagate unchanged. Package-detected source, format, and resource failures occur +before bytes are offered to a public sink. Once a sink method starts, arbitrary +short writes, disk failures, and user sink errors are not promised rollback. + +Error precedence is deterministic. A directly visible negative, contradictory, +or impossible structural field produces `FormatError` before dependent resource +work. A valid nonnegative declared size or count above a limit produces +`LimitError` before payload parsing or allocation. A detectable source invariant +mismatch produces `ArgumentError` before a later resource request. If detection +itself requires an allocation, the reserve-before-allocation `LimitError` wins. +Tests combine malformed-plus-over-limit and mutation-plus-over-limit inputs to +bind these rules. + +Test exact-limit success and limit-minus-one failure for materialized bytes, +metadata depth, container and schema width, physical leaf count, list and map +entries, prefix arrays, page count, row groups, and checked row, level, dense, +payload, offset, and frame arithmetic. Use virtual vectors and synthetic metadata +for overflow tests. Do not allocate multi-gigabyte inputs. The pinned large MAP +fixture proves rejection before large decompression or value materialization. +Wire-size and container-count limits use fixed expected values. Live materialized +memory minima are derived independently on each Julia runtime and then tested at +the derived minimum and one byte below it. Object-layout constants from one Julia +version are never used as a cross-version expectation. + +Every `Limits` field has an applicable read and write matrix: + +| Limit | Required paths | +|---|---| +| `max_footer_bytes` | Footer read, completed writer footer, preserved nested metadata. | +| `max_page_header_bytes` | V1/V2/dictionary/index/unknown page header read and writer header construction. | +| `max_page_bytes` | Compressed and uncompressed nested page read, page retry, dictionary page, and writer payload. | +| `max_page_index_bytes` | Cumulative nested OffsetIndex preflight, decode, and writer section. | +| `max_materialized_bytes` | Whole-table nested read, ordinary and provenance writes, test-model adapter, and rollback. | +| `max_schema_name_bytes` | Footer schema read, interning, ordinary write, and provenance rewrite. | +| `max_string_bytes` | Nested string/binary/logical JSON and BSON leaf read and write. | +| `max_decimal_bytes` | Nested DECIMAL conversion and fixed/byte-array read and write. | +| `max_container_elements` | Schema nodes, physical leaves, rows, entries, levels, pages, row groups, and indexes. | +| `max_metadata_depth` | Recursive schema, metadata, value conversion, reference model, reader, and writer. | + +Each applicable cell proves reserve-before-allocation, exact-bound success, +one-smaller failure, and cleanup. Tests also cover Thrift Int32 page/count fields, +Int16 row-group ordinals, and checked Int64 rows, offsets, ranges, sizes, and sums. +Exact schema-name registry tests run in isolated Julia subprocesses so prior global +interning cannot change their boundary. + +Production fixes follow minimized failing tests. Likely ownership is private +validation in `write_nested.jl`, shared with provenance validation only when the +same invariant applies. N5-C adds no public type or export. + +N5-C gate: + +- Every named mutation fails with the specified error class. +- Every exact boundary succeeds and its next smaller bound fails. +- Budget and output rollback tests pass on both Julia versions. +- Seeded hostile cases do not escape raw implementation errors. + +## N5-D: durable Java and Rust oracles + +Use this repository layout: + +```text +test/conformance/n5/manifest.toml +test/conformance/n5/golden/parquet-java/ +test/conformance/n5/golden/arrow-rs/ +test/conformance/n5/expected/ +test/conformance/n5/oracles/parquet-java/ +test/conformance/n5/oracles/arrow-rs/ +``` + +Each case records producer, version, commit, file SHA-256, flattened physical +schema, logical schema AST, ordered semantic pairs, leaf paths, maximum levels, +and exact repetition, definition, and dense-value streams. + +The external tools run in one Linux/amd64-only OCI image. Its base is Eclipse +Temurin 11.0.28+6 at Linux/amd64 manifest digest +`sha256:ab2527b3c9b7c15bc88f60dec19b2aa39939a6e0045fb8f538eeecbd7af59c69`. +The bootstrap installs Apache Maven 3.9.8 from the official archive whose tarball +SHA-512 is +`7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2`. +It installs Rust 1.96.1 from the channel manifest whose SHA-256 is +`87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975`. + +A networked bootstrap build resolves every Java GAV, records every artifact hash, +vendors the complete Arrow Rust source and crate graph, and records dependency +trees and checksums. It places the populated Maven repository and Cargo vendor +tree in the image. `test/conformance/n5/oracles.lock` records the resulting image +digest, Maven archive, Java artifact manifest, Rust channel manifest, Cargo vendor +manifest, and all upstream revisions. Updating it is a reviewed maintenance +action. + +The locked image reference is +`ghcr.io/juliaio/parquet-jl-n5-oracles@sha256:`, where `` is the +required 64-hex image value stored in `oracles.lock`. A clean CI host may use the +network only to pull that exact public reference. It verifies the pulled +`RepoDigest` against the lock before execution. It then runs the container with +`--network=none`. Thus image acquisition is networked, while Maven, Cargo, fixture +generation, and oracle execution are offline. + +The actual gate runs that verified image by digest with the network disabled. Maven uses +`--offline` and its image-owned repository. Cargo uses the checked `Cargo.lock`, +checked toolchain, `.cargo/config.toml` vendor replacement, `--offline`, and +`--locked`. A wrapper or lock file without the corresponding offline content is +not sufficient evidence. + +The bootstrap and gate interfaces are fixed: + +```sh +test/conformance/n5/bootstrap-oracles.sh --output n5-oracles.lock +test/conformance/n5/run-oracles.sh --lock n5-oracles.lock --network none +``` + +The bootstrap command may use the network and must reproduce the recorded +dependency manifests before a new digest is accepted. The run command must fail +if the image digest, corpus commit, fixture hash, toolchain, or offline dependency +is missing or different. + +The N5 workflow pins every GitHub Action by full commit SHA. It uses a named +Linux runner only to start the locked container; all oracle processes and tools +run inside the container. Pull requests run the offline locked gate. A weekly and +manual job performs a clean networked bootstrap, compares its dependency manifests +and image contents with the lock, and then runs the offline gate. Every job uploads +the manifest, logs, schemas, case IDs, and failure evidence with `if: always()`. + +The Java harness uses parsed physical schemas and raw `Group` verification for +legacy physical rows. It uses `AvroParquetReader` with +`AvroReadSupport` as its concrete high-level LIST/MAP semantic API. Inferred-schema +Avro checks run with +`-Dparquet.avro.add-list-element-records=false`; the default changes rule-5 +wrappers into record elements and does not match the binding scalar-element rows. +The setting is recorded in oracle evidence. The binding rule-3 fixture retains +the inner LIST annotation required by the specification example. Both inferred +Java Avro and Arrow Rust `RecordBatch` reads must return the exact nested-list +rows without an explicit schema that changes the physical interpretation. An +unannotated near-neighbor is reported under a different case ID; Java/Rust +high-level differences for that non-binding control are diagnostic. The harness +also uses +low-level column and page readers to report +exact repetition levels, definition levels, dense values, page versions, and row +counts. Avro maps require UTF-8 string keys. The Java harness therefore owns a +separately identified UTF-8-key direct LIST-of-MAP fixture whose inferred Avro +schema and rows prove the legacy LIST-over-MAP nesting. The independent Julia +golden retains the binding integer keys and exact integer streams. Java does not +authorize the integer-key domain; its high-level projection is used only for the +nested shape and has the specified last-value-wins map behavior. Logical map +adapters are otherwise secondary because they can collapse duplicate keys or +reject optional-key and key-only compatibility cases. + +The Rust harness uses its low-level Parquet writer for explicit repetition and +definition levels. Its low-level page and column reader reports the same physical +evidence. Arrow `RecordBatch` verification separately checks high-level schema and +row semantics where the Arrow type system can represent the layout. + +Both harnesses: + +1. Generate their owned fixtures. +2. Compare generated hashes and manifests with checked evidence. +3. Read Julia canonical files and schema-bearing rewrites. +4. Verify exact schemas, repetition and definition streams, dense values, page + versions, ordered physical values, logical values where representable, row + counts, and null and empty distinctions. +5. Emit machine-readable evidence with tool versions and case IDs. + +Oracle responsibilities are separate: + +| Fact | Required authority | +|---|---| +| Parquet 2.13 Thrift coverage, unknown fields, and raw-field re-emission | Julia generated-IDL and mutation tests. Parquet Java 1.17.1 embeds format 2.12 and is not an authority for post-2.12 fields. | +| Exact physical schema, page version, row count, repetition, definition, and dense streams | Independent Julia wire decoder plus Java and Rust low-level page/column evidence for their owned fixtures. | +| Canonical LIST/MAP logical rows | Java and Rust high-level readers where representable. | +| Optional-key, key-only, duplicate-key, and arbitrary-name physical layouts | Java and Rust low-level evidence plus ordered semantic manifests. | +| A disputed compatibility meaning such as direct LIST-of-MAP | At least one independent high-level Java or Rust interpretation; low-level agreement alone is insufficient. | +| Julia schema-bearing rewrite preservation | Julia exact metadata comparison plus external low-level schema and stream evidence. | + +The manifest binds external ownership for every mandatory compatibility family: + +| Mandatory cases | Required independent producer | Required readers and result | +|---|---|---| +| LIST rules 1 and 2 and both required/optional rule-5 forms | Parquet Java low-level fixture writer | Julia independent wire and production readers; Java low-level; Rust low-level where supported; Java Avro high-level normalized rows. | +| LIST rule 3 | Parquet Java low-level fixture writer using the specified inner LIST annotation, with an Arrow Rust low-level fixture as a cross-check | Julia independent wire and production readers; Java and Rust low-level; inferred Java Avro and Rust `RecordBatch` high-level exact rows. The unannotated near-neighbor has a separate case ID and diagnostic high-level outcomes. | +| Rule-4 `array` and `_tuple` | Parquet Java low-level fixture writer | Julia independent wire and production readers; Java and Rust low-level where supported; Java Avro high-level one-tuple rows. | +| Standard MAP and arbitrary entry/key/value names | Parquet Java low-level fixture writer | Julia independent wire and production readers; Java and Rust low-level; Java Avro high-level normalized rows. | +| Standalone outer `MAP_KEY_VALUE` | Parquet Java low-level fixture writer | Julia, Java, and Rust low-level readers where supported; Java Avro high-level map rows. | +| Duplicate keys | Arrow Rust low-level fixture writer | Julia, Java, and Rust low-level ordered pairs; Rust `RecordBatch` preserves entry order; last-value-wins projection checked separately. | +| Key-only MAP | Parquet Java low-level fixture writer | Julia, Java, and Rust low-level key order where supported; Julia checks the specified key-set or all-null-map meaning; Java Avro rejection and other high-level outcomes are diagnostic. | +| Compatibility-optional key with every key present | Pinned `incorrect_map_schema.parquet` plus Arrow Rust low-level fixture writer | Julia, Java, and Rust low-level where supported; high-level rejection or acceptance is recorded as diagnostic. | +| Direct legacy LIST-of-MAP | Parquet Java low-level fixture writer for both the binding integer-key physical case and a separately identified UTF-8-key semantic-authority case | Julia, Java, and Rust low-level where supported; inferred Java Avro on the UTF-8-key case must report `LIST` with exact null, empty, empty-map, duplicate-key last-value-wins, and present-map rows. The Julia integer-key golden retains exact ordered pairs and streams; Java does not authorize its key domain. | +| Actual-null optional key mutation | Test-owned mutation of the external optional-key fixture | Julia must reject eagerly; Java and Rust low/high-level outcomes are diagnostic and do not make the invalid file valid. | + +Every valid serialized gap has at least one external producer that does not use +the Julia reference model. A harness API that cannot expose one physical form is +recorded as unsupported evidence for that harness, not silently passed. The other +required producer and reader obligations remain binding. + +Julia reads every Java and Rust fixture. Java and Rust read every Julia fixture +that their public or low-level API can represent. Low-level success proves the +physical contract only. A compatibility case that claims a disputed high-level +meaning also needs one independent high-level oracle to report that meaning; it +otherwise remains open. + +Oracle jobs run in CI with the locked OCI image and networking disabled. They do +not silently skip when Java, Rust, the corpus, or a fixture is absent. PyArrow +25.0.1 and DuckDB 1.5.5 remain secondary triage oracles, not substitutes for the +required Java and Rust evidence. + +N5-D gate: + +- All required Java and Rust fixtures have checked hashes and semantic manifests. +- Julia reads and rewrites them exactly. +- Both external harnesses verify Julia output without an absence skip. +- CI preserves case IDs, versions, schemas, hashes, and failure evidence. + +## Final N5 gate + +N5 is complete only when: + +- N5-A through N5-D are green on their required platforms. +- The thirteen materializable pinned nested fixtures pass exact read and rewrite + checks, and the large MAP fixture passes its bounded schema, small-leaf, and + pre-allocation rejection gate. +- Julia 1.10 and current stable pass the complete suite with the pinned corpus. +- The external oracle jobs pass from clean environments. +- Independent review finds no open high-severity correctness or resource issue. +- Only the Stage 4 nested-data ledger row moves to complete. + +Passing N5 closes nested conformance. It does not make the complete Parquet 1.0 +roadmap release-ready. diff --git a/docs/dev/n6-statistics-plan.md b/docs/dev/n6-statistics-plan.md new file mode 100644 index 0000000..a5309c0 --- /dev/null +++ b/docs/dev/n6-statistics-plan.md @@ -0,0 +1,385 @@ +# N6-A trusted statistics and column order plan + +N6-A implements one private truth layer for row-group column statistics and +adds conforming statistics to new files. It does not prune any data. Column +indexes, bloom filters, scan predicates, and page pruning remain separate +slices. + +## Review state + +Two independent read-only audits selected this slice as the next actionable +Stage 5 boundary. Both found that later pruning work needs one trusted order and +comparison model first. They also agreed that connecting an unreviewed model to +pruning could silently remove matching rows. + +The requested Claude Fable 5 Max review remains unavailable because the account +reported an insufficient credit balance. No Claude approval is claimed. This +plan must receive an exact read-only review before production files change. + +## Prerequisites and incomplete evidence + +The local schema and footer hardening phase is complete. The full package suite +passes on Julia 1.10 and 1.12. The canonical offline Java and Rust gate passes +352 files with 254 paired mappings. + +This does not close the durable N5 release gate. No published oracle digest or +`oracles.lock` exists, CI does not run the locked oracle image, and the feature +ledger still records nested data as in progress. Publishing an image and +accepting a lock require separate authorization. N6-A may proceed without +changing frozen N5 evidence, but no Stage 4 release-complete claim is allowed. + +LZO is also a separate format-completeness blocker. The available registered +implementation is not license-compatible with this MIT core. N6-A does not +change that boundary. + +## Pinned authority + +The normative source is Apache Parquet format 2.13.0 at commit +`c47e2a66e88943fc46fde1b028a9432f14fdf5c0`. + +The implementation follows these rules from the pinned IDL and logical-type +specification: + +- `min_value` and `max_value` have no defined meaning without a complete, + leaf-aligned `column_orders` vector. +- Deprecated `min` and `max` always use signed comparison, independent of + `column_orders`. +- Bounds use PLAIN bytes. A BYTE_ARRAY bound omits its length prefix. +- STRING, ENUM, UUID, JSON, BSON, raw BYTE_ARRAY, and raw fixed byte arrays use + unsigned byte-wise order. +- Signed INTEGER, DATE, TIME, TIMESTAMP, and the matching physical integers use + signed order. Unsigned INTEGER uses unsigned numeric order. +- DECIMAL compares the represented signed unscaled value. Fixed and variable + byte-array DECIMAL values therefore do not use raw unsigned byte order. +- INTERVAL, UNKNOWN, VARIANT, GEOMETRY, GEOGRAPHY, LIST, MAP, and stable INT96 + have no type-defined min/max order. +- FLOAT, DOUBLE, and FLOAT16 may use `TYPE_ORDER`, but writers should use + `IEEE_754_TOTAL_ORDER` for defined NaN, signed-zero, and payload ordering. +- A floating writer always emits `nan_count`. With IEEE total order, mixed + bounds exclude NaNs. An all-NaN non-null set uses its total-order NaN extrema. +- Missing counts remain unknown. A present zero remains known. + +Producer-version trust is compatibility policy, not wire-format authority. The +first checked rule is parquet-mr PARQUET-251. It applies only to physical +BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY bounds. Null, empty, or wholly unparsable +`created_by` values are untrusted for those bounds. A parsed non-parquet-mr +application is unaffected. A parquet-mr value without a semantic version is +untrusted. A parquet-mr version below 1.8.0 is untrusted, except for versions +greater than or equal to `1.5.0-cdh5.5.0` and less than `1.5.0`. Version 1.8.0 +and later passes this rule. Parse failure is nonfatal, and counts remain +independent. + +N6-A deliberately applies this conservative PARQUET-251 decision to both +modern and deprecated bound families. Pinned parquet-java applies the check in +its deprecated-bound fallback. Applying it to both families cannot create a +false exclusion and avoids trusting unusual modern bounds carrying an affected +producer identity. More producer rules enter only with pinned primary-source +evidence and direct boundary tests. A remembered cutoff is not sufficient. + +A second compatibility rule is pinned to Apache Arrow commit +`515410b2a14ac766258e00b07eab9e5ee2692a62`, `cpp/src/parquet/metadata.cc`. +Parquet-cpp before 1.3.0 and parquet-mr before 1.10.0 produced unsafe +statistics when the selected comparator order was not signed; IEEE total order +is also non-signed for this rule. For either producer, a missing usable version +is conservatively in the affected range. Pinned Arrow compares only the numeric +version triplet. N6-A is deliberately stricter and treats prereleases of 1.3.0 +and 1.10.0 as affected. Both bounds in the selected family become unknown +unless both are present with identical raw bytes, where order cannot change the +result. Apply this rule to both bound families. Apply PARQUET-251 independently +as the stronger rule for affected BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY bounds. +The equality exception never overrides PARQUET-251. Test both release-candidate +and final cutoff strings, including an affected producer paired with IEEE order. + +## Scope + +### Reader truth layer + +Add `src/statistics.jl`. It defines private order, bound, count, and trust +states. It must not export a new name. + +The layer receives the validated schema, file `created_by`, the leaf-aligned +column order, and `ColumnMetaData`. It returns these +facts independently: + +- lower bound; +- upper bound; +- null count; +- NaN count; +- distinct count; +- bound exactness; +- declared order; +- producer trust decision and reason. + +One absent or unusable fact does not erase unrelated valid facts. For example, +an absent lower bound does not erase a known null count. + +Use these count and order rules: + +- Validate `null_count`, `nan_count`, and `distinct_count` independently as + nonnegative and no greater than `ColumnMetaData.num_values`. Never compare a + nested leaf count with the row-group row count. +- `nan_count` is legal only for FLOAT, DOUBLE, and FLOAT16. On another leaf it + is malformed metadata. When null and NaN counts are both present, add them + with checked arithmetic and require the sum to be no greater than + `num_values`. +- When null and distinct counts are both present, require `distinct_count` to + be no greater than `num_values - null_count`. +- A negative count, an invalid checked count relationship, a short or long + `column_orders` vector, IEEE total order on a non-floating leaf, or an invalid + fixed physical width is malformed metadata and raises `FormatError` during + explicit statistics validation. +- An absent `column_orders` vector makes modern bounds unknown. It does not make + the file unreadable. +- An unknown future `ColumnOrder` member is preserved by the generated metadata + layer and makes only that leaf's modern bounds unknown. +- `TYPE_ORDER` is a legal leaf-aligned placeholder when a leaf has no defined + order. Its bounds are ignored. LIST and MAP annotations do not suppress the + independently defined order of their descendant physical leaves. +- A producer known to have corrupt statistics makes the affected bounds + unknown. Counts that are independently valid stay available. + +Use these floating rules: + +- A count arithmetic or domain violation always raises `FormatError`; it never + degrades only the bounds. When known counts prove that no non-null value + exists, every bound is unknown. +- Under floating `TYPE_ORDER`, ignore each NaN bound independently. Widen a + `+0.0` lower bound to `-0.0`, and widen a `-0.0` upper bound to `+0.0`. + Downgrade exactness on each widened side. +- Under `TYPE_ORDER`, counts that prove a nonempty all-NaN set make any present + bound a format contradiction and invalidate both bounds in the selected + family. With insufficient counts, ignore a NaN bound independently and keep + an independently valid non-NaN bound. +- Under `IEEE_754_TOTAL_ORDER`, use a raw-bit total-order key. Do not use Julia + `isless`. For only `+0.0`, both extrema are `+0.0`. For only `-0.0`, both are + `-0.0`. When both occur, the lower bound is `-0.0` and the upper bound is + `+0.0`. +- IEEE NaN bounds are trusted only when known null and NaN counts prove that a + nonempty set of non-null values is entirely NaN. That state permits only NaN + bounds. The extrema are the smallest and largest raw NaN patterns actually + present. Never synthesize sentinel NaNs. +- An IEEE state with at least one proven non-NaN value permits only non-NaN + bounds. A bound-kind contradiction in either side invalidates both bounds in + the selected family. With insufficient counts, ignore each NaN bound + independently and keep an independently valid non-NaN bound. + +Use these bound-family and value rules: + +- Treat modern and deprecated bounds as separate families. If either modern + side is present, use only modern fields and never fill a missing modern side + from deprecated metadata. Use deprecated fields only when both modern fields + are absent, their signed order matches the leaf meaning, and the producer is + trusted. A contradiction invalidates both bounds in the selected family. +- Validate one-sided bounds independently. An absent bound ignores its + exactness flag. A present modern bound with an absent, false, or true flag has + unknown, inexact, or exact status, respectively. +- Require exact PLAIN widths for BOOLEAN, INT32/FLOAT, INT64/DOUBLE, INT96, + FLOAT16, and fixed byte arrays. A BOOLEAN byte must encode zero or one. +- Raw BYTE_ARRAY and fixed-byte bounds may contain arbitrary bytes. STRING and + ENUM require valid UTF-8. JSON and BSON require valid documents. UUID and + FLOAT16 require exactly 16 and 2 bytes. DECIMAL must fit its declared + precision. Annotated integer bounds must fit their declared bit width. TIME + bounds must fit the declared daily domain. +- A semantically invalid optional logical bound becomes unknown while its raw + metadata stays preserved. A structurally wrong fixed width remains a + `FormatError`. No unusable bound becomes evidence for exclusion. +- INT96 order remains undefined in N6-A. The format's recommended legacy + comparator does not make it a stable type-defined order. + +Add `Limits.max_statistics_value_bytes::Int64 = 4096`. It is a per-raw-bound +interpretation and writer-emission limit. Equality succeeds. A reader bound one +byte over remains footer-owned and preserved, but becomes unknown without a +copy, semantic parse, or `LimitError`. A writer bound one byte over omits both +bounds, but retains counts and column order. Check fixed-width structure before +the policy limit. Check an oversized variable bound before UTF-8, JSON, BSON, +or DECIMAL work. A negative configured maximum fails deterministically. Zero +disables nonempty bounds. A negative value raises `ArgumentError`. Validate it +at the reader and writer entry points before reader work, writer callbacks, or +destination mutation. `max_materialized_bytes` remains the cumulative live +allocation limit. + +Implement PLAIN bound decoding without constructing a full column. Fixed +physical widths must match exactly. Variable BYTE_ARRAY bounds are the raw +bytes. Keep reader byte bounds as footer-owned references. Avoid BigInt in +DECIMAL comparison by comparing normalized signed two's-complement bytes. +Precharge each reader scratch allocation. Use one production comparator seam +shared by the reader, writer, and later ColumnIndex work. The independent test +model must use separate code, types, and decoding logic. + +### Writer statistics + +Add `src/write_statistics.jl` and a `statistics::Bool=true` keyword to both +public `Parquet.write` methods and `_encodefile`. + +Compute each summary from the already validated, row-group-sliced +`WriteLeafPlan`. Do not call the source table, user vectors, map callbacks, or +conversion hooks again. Work is linear in present values. Fixed-width +accumulation allocates no memory per value after warm-up. Start the summary pass +only after `_nestedwritefields` has completed its terminal source barrier. The +row-group leaf values are then operation-owned and safe from later source +mutation. + +When `statistics=true`: + +- always emit an exact `null_count` as the leaf-entry count minus its dense + non-null value count; +- emit `nan_count` for FLOAT, DOUBLE, and FLOAT16, including zero; +- emit modern `min_value` and `max_value` with exactness flags when the leaf has + a defined order and at least one value that the selected order can bound; +- use `IEEE_754_TOTAL_ORDER` for FLOAT, DOUBLE, and FLOAT16; +- use `TYPE_ORDER` for all other leaves in the complete leaf-aligned + `column_orders` vector; +- emit no min/max for undefined-order leaves; +- emit no deprecated `min` or `max` fields; +- omit both bounds when an exact encoded bound exceeds + `max_statistics_value_bytes`; keep valid counts and column order; +- encode zero as the actual extrema under raw-bit IEEE total order. Only + `+0.0` produces two `+0.0` bounds. Only `-0.0` produces two `-0.0` bounds. + Both signs produce `-0.0` below `+0.0`; +- count NaNs only among dense non-null values. For an all-NaN non-null set, + emit the smallest and largest NaN bit patterns actually present under IEEE + 754 total order; +- pass the live writer budget through statistics and `column_orders` + construction. Reserve before retained bound-vector and metadata allocations, + and restore the exact starting charge on failure. + +When `statistics=false`, preserve the current no-statistics file behavior and +omit `column_orders`. Skip the summary scan and all related allocations. The +central N5 `n5productionencodedbytes` options and every frozen N5 output path +must pass this option explicitly. Add a forbidden-default scan. Do not rewrite +frozen fixtures or manifests. + +Page-header statistics remain absent. N6-B will add complete ColumnIndex page +bounds and will decide whether page-header duplication provides enough value. +`SizeStatistics`, histograms, bloom filters, and geospatial statistics are not +part of N6-A. + +## Tests + +Add `test/statistics.jl` and `test/write_statistics.jl`. Include both from +`test/runtests.jl`. Add a separate `test/conformance/n6/` area. Do not edit the +N5 manifests, expected evidence, or golden files. + +The focused matrix covers: + +- every supported physical type and stable logical order; +- signed and unsigned integer disagreements; +- DECIMAL in INT32, INT64, BYTE_ARRAY, and FIXED_LEN_BYTE_ARRAY; +- raw and logical byte-wise types, embedded NUL, invalid UTF-8 raw bytes, and + exact fixed widths; +- all 65,536 FLOAT16 bit patterns plus sampled Float32 and Float64 infinities, + signs, zeros, subnormals, quiet and signaling NaNs, and distinct payloads; +- only positive zero, only negative zero, both zeros, mixed finite and NaN, + all-NaN, missing-count, and contradictory-count states; +- empty, all-null, all-NaN, and mixed null, NaN, and ordinary values; +- absent, one-sided, inexact, oversized, contradictory, and wrong-width bounds; +- absent, short, long, unknown, and illegal column orders; +- absent, zero, negative, excessive, and contradictory counts; +- valid and invalid `distinct_count`, and illegal non-floating `nan_count`; +- trusted, affected, fixed, malformed, absent, and unrelated `created_by` + strings, including exact 1.8.0 and CDH endpoints and prereleases; +- the cross-product of modern and deprecated family presence, parquet-cpp + 1.3.0 and parquet-mr 1.10.0 cutoffs, PARQUET-251, signed versus non-signed + order, and identical versus distinct encoded bounds; +- nested leaf-entry counts and row-group slices; +- negative, zero, exact-limit, and one-byte-over statistics limits, including + fixed-width-before-limit and variable-limit-before-semantic precedence; +- mutation between planning and emission; +- budget rollback and unchanged IO/path destinations on failure. + +Add two-row-group no-pruning fixtures whose page bodies are identical while +statistics are absent, trusted, producer-untrusted, oversized, and semantically +unusable. Full `Table` values and pre-footer body-range reads must be identical +for every variant. Capture an instrumented production-source range trace and +read count, not only byte equality. N6-A must perform no statistics-based +row-group or page selection and must add no page-header statistics. + +Freeze the independent model and case manifest before production code changes. +The model must not call the production comparator or bound decoder. Add a scan +that forbids such calls. For every emitted file, it proves that each applicable +decoded value is contained by its bounds and that each exact bound equals the +independent extremum. Seeded metadata mutations may fail or become unknown. +They must never produce a false trusted bound. + +Add warmed allocation probes for zero and many row groups. Fixed-width summary +work must not allocate per value after warm-up. Retained growth may be +output-sized only. + +Use the pinned corpus statistics fixtures, including: + +- `floating_orders_nan_count.parquet`; +- `nan_in_stats.parquet`; +- `single_nan.parquet`; +- `float16_zeros_and_nans.parquet`; +- `float16_nonzeros_and_nans.parquet`; +- `binary_truncated_min_max.parquet`; +- `int96_timestamp_order.parquet`; +- the INT32, INT64, byte-array, fixed-byte, and DECIMAL statistics fixtures. + +## Independent interoperability + +Create N6-owned fixtures and evidence. Do not extend or regenerate frozen N5 +evidence in place. + +- Freeze an N6 capability matrix, source/toolchain manifest, fixture manifest, + and evidence schema before production code changes. This is local reviewed + test evidence, not publication of an oracle image or the repository + `oracles.lock`. +- Generate a test-owned non-Julia Java Compact-Thrift footer scanner directly + from the exact pinned Parquet 2.13 IDL with an Apache Thrift compiler and + runtime whose versions and artifact hashes are fixed by the N6 manifest. + Keep it separate from Parquet.jl's generated metadata, comparator, and + production decoder. Run it in a separate JVM process with its own classpath + so its generated `org.apache.parquet.format` classes cannot collide with + parquet-java's embedded 2.12 classes. It inspects raw `column_orders`, bounds, + exactness flags, counts, signed-zero bits, and NaN payload bits. Record these + raw assertions separately from parquet-java's embedded 2.12 semantic API. + Label this as raw 2.13 wire evidence, not parquet-java semantic support. +- Combine the raw scanner with the frozen independent model and pinned Apache + 2.13 statistics corpus for IEEE semantic evidence. No unsupported result may + count as a pass. +- Parquet Java generates and checks readability, values, TYPE_ORDER, legacy + bounds, and producer-version compatibility. Its pinned high-level API does + not authorize IEEE total order or `nan_count`. +- Arrow Rust generates and checks readability, values, and supported + TYPE_ORDER metadata. Its pinned metadata layer treats IEEE order as unknown + and lacks field 9, so it does not authorize IEEE order, `nan_count`, or NaN + payloads. +- PyArrow verifies logical values and its exposed row-group statistics. +- DuckDB verifies file readability and its Parquet metadata view. +- Every producer, toolchain, source revision, fixture, and evidence file is + pinned and hashed. + +The existing offline N5 gate is rerun unchanged after N6-A. The new N6 oracle +gate remains separate until its own image and lock are reviewed and authorized. + +## Acceptance gates + +1. Obtain two read-only plan reviews with no open P0 or P1 disagreement. +2. Freeze and review the independent model, case manifest, capability matrix, + raw 2.13 scanner pins, and evidence schema. Do not edit production files + before this gate passes. +3. Implement the reader truth layer and pass focused Julia 1.10 and 1.12 tests. +4. Obtain an exact-hash read-only review of the reader layer. +5. Implement writer statistics and pass focused Julia 1.10 and 1.12 tests. +6. Obtain an exact-hash read-only review of the writer layer. +7. Pass the independent N6 model and capability-specific Java, Rust, PyArrow, + DuckDB, raw 2.13, and Apache corpus evidence. +8. Pass the complete N5 and package suites on Julia 1.10 and 1.12. +9. Pass no-pruning/read-count evidence, bounds-checking, generator freshness, + warmed allocation probes, + `git diff --check`, and forbidden-copy scans. +10. Rerun the canonical offline N5 Java and Rust gate with no change to frozen + evidence. + +Document the public `statistics` keyword and +`Limits.max_statistics_value_bytes`, including its 4096-byte default, before +N6-A closes. + +## Completion boundary + +N6-A closes trusted row-group statistics and writer column-order production +only. It does not close Stage 5. ColumnIndex content, bloom filters, +`Tables.Scan`, row-group or page pruning, LZO, encryption, Variant, geospatial +support, Dataset behavior, documentation beyond the required N6 API additions, +performance, CI publication, PkgEval, and release gates remain incomplete. diff --git a/docs/dev/nested-plan.md b/docs/dev/nested-plan.md new file mode 100644 index 0000000..c55ce6b --- /dev/null +++ b/docs/dev/nested-plan.md @@ -0,0 +1,637 @@ +# Stage 4 nested data agreement + +This document defines the complete nested reader and writer design for Parquet.jl +1.0. It refines the Stage 4 roadmap. It is an implementation agreement, not a +claim that the work is complete. + +The format target is Apache Parquet 2.13.0. The annotated tag ultimately peels to +source commit `c47e2a66e88943fc46fde1b028a9432f14fdf5c0`. The reader accepts all +specified legacy nested encodings. New ordinary writes use canonical modern +schemas. + +## Review disposition + +The design has four independent inputs: + +1. The Parquet 2.13 nested-type rules and pinned Apache test corpus. +2. A reader review of recursive Dremel reconstruction. +3. A writer review of schema ownership and recursive shredding. +4. A representation review against the Arrow rewrite and Tables.jl. + +A fresh Claude Fable 5 Max review was requested. The CLI returned `Credit balance +is too low`. No fresh Claude approval is claimed for this phase. Independent Codex +reviews supply the available adversarial gate. Production edits start only after +those reviewers accept this exact plan. + +## Fixed decisions + +- Physical page decoding continues to return one `LeafStream` per physical leaf. +- Raw `SchemaNode` trees remain an exact physical view. +- A second, context-aware plan normalizes LIST, MAP, struct, and repeated forms. +- Nested storage is package-owned and columnar. +- Nested scalar access uses non-allocating package-owned views. +- Plain Julia containers remain first-class writer input. +- `missing` is the only null value. `nothing` is not a null alias. +- Modern logical annotations take precedence over legacy converted annotations. +- New output uses canonical three-level LIST and MAP schemas. +- A schema-bearing `Parquet.Table` preserves every representable source schema + when rewritten. Zero-field groups remain read-only until an independent writer + accepts them. +- Nested vectors are structurally read-only in this phase. +- No new names are exported. + +## Schema normalization + +Add private semantic plan nodes for leaves, structs, lists, and maps. Each plan +stores its source node, parent and present definition thresholds, repeated-entry +level when present, ordered children, and contiguous descendant-leaf range. + +Plan compilation happens before page reads or output allocation. It validates the +whole topology under `Limits`. It does not mutate the raw schema. + +The message root and primitive nodes cannot carry LIST, MAP, or MAP_KEY_VALUE. +Reject those placements before group classification. Any modern annotation, +including an unknown future member, blocks every legacy fallback. This includes +legacy MAP_KEY_VALUE fallback. + +The reader accepts an otherwise ordinary message root marked REQUIRED because +the pinned parquet-cpp corpus uses that historical form. It rejects OPTIONAL, +REPEATED, and unknown root repetition markers. Canonical output omits the marker. + +Group annotation classification follows these rules: + +| Group metadata | Semantic result | +| --- | --- | +| Modern LIST | LIST; validate LIST topology. | +| Modern MAP | MAP; validate MAP topology. | +| Other known modern type plus legacy LIST or MAP | Modern metadata wins; reject an invalid group/type combination. | +| Unknown modern type plus legacy LIST, MAP, or MAP_KEY_VALUE | Preserve as an ordinary group; do not apply legacy fallback. | +| No modern type plus converted LIST | Legacy LIST. | +| No modern type plus converted MAP | Legacy MAP. | +| Standalone converted MAP_KEY_VALUE | MAP compatibility alias. | +| Unannotated REPEATED outside LIST or MAP | Required list of required elements. | + +Nested duplicate struct field names remain ordered and valid. Name-based access is +ambiguous, but positional access and schema-preserving rewrites remain exact. +Top-level duplicate names remain unsupported by `Parquet.Table` because its Tables +column object is a `NamedTuple`. Low-level schema and leaf APIs can still inspect +such a file. + +A required leafless struct can be synthesized from its observable parent +occurrence count. An optional or repeated leafless group is rejected because its +presence or cardinality is not represented in any leaf stream. + +A LIST, MAP, or MAP_KEY_VALUE group cannot normally be REPEATED. A repeated LIST +or MAP-compatible group, including a standalone MAP_KEY_VALUE alias, is accepted +only when it is the direct element selected by a legacy two-level LIST rule. Its +REPEATED marker belongs to the parent list, so the nested collection is a required +logical element. Reject every other repeated annotated collection. + +One separate case is required inside an already classified MAP. Its sole entry +group must be REPEATED and may carry converted MAP_KEY_VALUE. That annotation is +only an entry marker. It does not create another map and is not subject to the +standalone placement rule. + +### LIST compatibility + +The repeated child of a LIST group is interpreted in this exact order: + +1. A primitive is the required element. +2. A group with two or more fields is the required struct element. +3. A group with one REPEATED child is the required element. This preserves nested + legacy lists. +4. A one-field group named exactly `array` is the required one-field struct + element. +5. A one-field group named exactly `_tuple` is the required one-field + struct element. +6. Any other one-field group is unwrapped. Its child repetition controls element + nullability. + +A zero-field repeated wrapper is invalid. Compatibility names are exact and +case-sensitive only in rules 4 and 5. Other wrapper and element names are ignored. + +The canonical writer emits: + +```text + group field (LIST) { + repeated group list { + element; + } +} +``` + +The outer group carries modern LIST and converted LIST metadata. + +### MAP compatibility + +A MAP has one repeated entry group. The first entry child is the key. The optional +second child is the value. Names are not semantic. + +- The key must be REQUIRED in canonical data and ordinary inferred output. +- An OPTIONAL key schema is accepted for the documented Presto, Trino, and Athena + compatibility case. +- An actual null key is always invalid. +- A compatible OPTIONAL-key source still exposes a nonmissing key element type. + Optionality stays only in source-schema provenance. +- A REPEATED key or value is invalid. +- A value may be REQUIRED, OPTIONAL, or omitted. +- An omitted value is exposed as `missing` while the schema retains that the child + did not exist. +- Duplicate encoded keys remain ordered and preserved. +- Explicit conversion to `Dict` applies last-encoded-value-wins behavior. +- Entry groups with zero or more than two children are invalid. + +The canonical writer emits: + +```text + group field (MAP) { + repeated group key_value { + required key; + value; + } +} +``` + +The outer group carries modern MAP and converted MAP metadata. The canonical +middle group does not carry MAP_KEY_VALUE. + +Ordinary writing rejects every zero-field group, including `NamedTuple{()}` and an +empty message. A schema-bearing rewrite also rejects a zero-field group until an +independent implementation proves an interoperable write. The reader may still +synthesize a required leafless struct when its occurrence count is observable. + +## Runtime vectors + +The package owns three read-only columnar containers: + +- `ListVector`: zero-based offsets, optional validity, and one child vector. +- `StructVector`: ordered runtime names, an optional compact rank map, and child + vectors aligned to present struct occurrences. +- `MapVector`: zero-based offsets, optional validity, a key vector, and an optional + value vector. + +Offsets use `Int32` until the flattened child count requires `Int64`. Constructors +check nonnegative monotonic offsets, terminal offsets, validity lengths, child +lengths, and checked conversion to `Int`. A null list or map has an empty span. A +present empty list or map also has an empty span. Validity is the only distinction. + +An optional `StructVector` stores a zero-based prefix-rank vector with one entry +per logical row plus one. Each adjacent difference is zero for a null struct or +one for a present struct. Every child length equals the last rank. A required +`StructVector` omits the rank vector and every child length equals the struct +length. This keeps required child element types exact. It never fabricates a +required value under a null ancestor. Constructors, row-group concatenation, and +schema-bearing writes use this same compact invariant. The rank starts at zero and +uses the same checked `Int32`-to-`Int64` promotion as list and map offsets. + +Indexing returns: + +- `missing` for a null container or struct; +- `ListValue <: AbstractVector` for a present list; +- `StructValue` for a present struct; and +- `MapValue <: AbstractVector{<:Pair}` for a present map. + +The views retain their owner and do not allocate the nested row. `collect` and +`copy` follow normal shallow Julia semantics. They remove only the outer package +view. A nested child view stays a view until the caller also collects it. + +`StructValue` supports positional access. String or Symbol access succeeds only +when one field has that name. It throws for an absent or duplicate name. Ordered +pair iteration preserves empty and duplicate names. Explicit conversion to a +`NamedTuple` requires unique valid names. + +`MapValue` is an ordered physical view. It indexes encoded entry positions and +iterates ordered pairs. It does not claim multimap semantics. It is not an +`AbstractDict`, because that contract cannot retain duplicate keys. The namespaced, +unexported `Parquet.maplookup(value, key[, default])` compares key content and +returns the last encoded match. Array `getindex` and `get` remain positional, so an +integer logical key cannot conflict with an encoded entry index. + +`Dict(value)` is an explicit logical conversion. Package views implement recursive +content-based `isequal` and `hash` where the logical key has stable Julia hash +semantics. This includes supported scalar keys and recursive list or struct keys +whose descendants have stable content semantics. Conversion rejects a map-valued +key or any other unsupported composite key shape with an actionable error while +the ordered view remains lossless. Byte-array keys are copied before insertion. +Mutating a key obtained from the returned dictionary has Julia's normal unsafe +dictionary-key behavior. + +Nested wrapper types stay namespaced and unexported. Tables.jl treats them as +scalar column elements. `Tables.columns(table)` remains the top-level `NamedTuple`. +`Tables.schema(table)` reports the exact nested wrapper element types, including +missingness. Nested wrappers do not themselves claim a Tables table or row contract. + +## Reader algorithm + +The reader has three layers: + +1. Compile the raw schema into semantic plans. +2. Decode every descendant leaf once into a `LeafStream`. +3. Zip the aligned streams into package-owned vectors. + +The zipper uses one cursor per descendant leaf. It does not trust one leaf as an +unchecked structural driver. For a node whose parent repetition level is `P`, a +level `rep > P` continues the current parent occurrence and `rep <= P` starts the +next occurrence. Deeper repeated levels are projected away before siblings are +compared. The zipper runs two passes: + +1. Validate occurrence boundaries, optional presence, repetition projection, + counts, limits, and final offsets. +2. Convert dense scalar values and fill exact-sized buffers. + +State tests are relative to the parent plan: + +- A definition below the parent threshold is an absent-ancestor placeholder. +- An optional node below its present threshold is null. +- A list or map below its repeated-entry threshold is empty. +- A list or map at or above that threshold has one or more entries. +- Required descendants under a null ancestor are placeholders, not corrupt data. +- A null or empty collection has one placeholder and no continuation. + +For every shared struct or collection, all descendant leaves must agree on the +collapsed occurrence boundaries and node presence. Deeper repeated nodes are +collapsed before sibling repetition is compared. + +Final validation proves: + +- every top-level row begins at repetition zero; +- the metadata row count was assembled; +- every level entry was consumed once; +- every dense physical value was consumed once; +- no continuation follows a null or empty collection; and +- an optional-schema map key is present for every encoded entry. + +The existing flat single-leaf path remains the fast path. Nested assembly happens +per row group. Parts concatenate once with checked offset rebasing. + +## Writer architecture + +The writer separates file schema ownership from physical leaf chunks. One +`WritePlan` owns: + +- the flattened schema exactly once; +- the parsed `Schema` self-check; +- the top-level row count; and +- a `Vector{WriteRowGroupPlan}`. Each row group contains one chunk per physical + leaf. N1 emits one row group for nonzero input and none for zero-row input. + +The schema root child count is the number of top-level fields, not the number of +leaves. Leaf order is `schema.leaves` order. + +Private recursive writer plans mirror leaf, struct, list, and map nodes. Ordinary +input inference uses declared types only: + +1. Recognize supported scalar and logical types first. +2. Infer a struct only from a concrete `NamedTuple`. +3. Recognize package `StructVector`/`StructValue`, `ListVector`/`ListValue`, and + `MapVector`/`MapValue` by semantic kind at every recursion depth. Map recognition + occurs before the general vector rule. +4. Infer a map from a concrete `AbstractDict{K,V}`. +5. Infer a list from a concrete `AbstractVector{E}`, except byte vectors. +6. Use `Union{Missing,T}` for optionality at every non-key level. +7. Require a nonmissing MAP key type and reject every actual missing key. +8. Reject `Any`, unresolved abstract containers, heterogeneous structural unions, + and unparameterized containers. +9. Do not inspect present values to guess structure. + +This makes zero-row and all-null typed input deterministic. Scalar metadata that +requires aggregation, such as decimal precision, may still scan values. Ambiguous +all-null nested logical leaves use one explicit, namespaced recursive schema +wrapper in N3. That API gets a focused review and adds no exports. + +### Recursive shredding + +Allocate one mutable builder per physical leaf. Use a count pass followed by an +emit pass. The count pass performs checked arithmetic and all possible validation +before allocation. Concurrent mutation is unsupported. The emit pass revalidates +every safety-critical shape, count, scalar constraint, and resource bound. It +rejects a mutation that violates those facts. It does not claim to detect a +same-shape value change. + +- A missing required node is an error. +- A missing optional node emits one absent marker to every descendant leaf. +- A present optional node increments definition before descent. +- A required struct descends into every child. +- An empty repeated node emits one empty marker to every descendant leaf at the + current definition level, before the repeated increment. +- Every present repeated item increments definition once before descent. +- The first repeated item keeps the incoming repetition level. +- Later siblings use that repeated node's repetition level. +- A required leaf emits one level entry and one dense value. +- A missing optional leaf emits only its level entry. +- A present optional leaf increments definition and emits its dense value. + +Every null or empty ancestor emits a marker to every descendant leaf. Each finished +builder passes through `LeafStream(...; expected_rows=rows)`. + +### Schema-bearing rewrites + +`Parquet.write(table::Parquet.Table)` treats `table.metadata.schema` as the +authoritative flattened schema. At the start of each write, it reparses those +elements into a fresh operation-owned `Schema`. It recursively compares that +fresh tree with `table.schema`, including every element, path, level, leaf +ordinal, child edge, and leaf ordering. It rejects any disagreement. All later +binding, path selection, level calculation, and shredding use only the fresh +tree. This prevents mutable `SchemaNode` vectors from changing the physical +write plan. + +The provenance path does not re-infer legacy structure from element types. It +preserves: + +- exact repetition shape and physical paths; +- source names and field IDs; +- logical and converted annotations; +- unknown Thrift fields and unknown modern annotations; +- legacy two-level and special-name LIST forms; +- unannotated repeated fields; +- standalone and middle MAP_KEY_VALUE forms; +- optional-key schema compatibility; and +- omitted map values. + +The writer revalidates the vector tree against the fresh schema before both +passes. `table.rows` must equal every top-level vector length in both passes. A +legacy zero `FileMetaData.num_rows` sentinel is accepted only as source history; +the new footer records the actual `table.rows` value. Top-level duplicate names +remain outside the `NamedTuple` table boundary. Duplicate nested struct names +remain ordered and representable. + +Each logical leaf converts with its exact preserved source `SchemaElement`. +Neither schema construction nor value conversion calls `_canonicalwriteelement`. +The count pass validates the exact logical-to-physical conversion and payload +size. The emit pass allocates and charges the exact dense physical vector and +each copied variable-width payload. Any semantic group with no physical leaf is +rejected before a level or dense buffer allocation. + +`Parquet.write(Tables.columntable(table))` is an ordinary detached write and emits +canonical schemas. + +### Leaf encoding and page boundaries + +Internal leaf identity is always the physical schema leaf ordinal plus its exact +emitted path. A public encoding override may use an exact path tuple only when that +path selects one leaf. An ambiguous path is an error. A positive integer selects a +physical leaf ordinal and remains available when duplicate sibling names create +identical paths. For example: + +```julia +(:orders, :list, :element, :price) => :delta_binary_packed +(:attributes, :key_value, :value) => :dictionary +``` + +Path tuple segments may be Symbols or Strings and normalize to Strings. Reject two +override keys that normalize to the same selector. Symbol and String keys remain +valid for flat one-segment columns. A top-level group alias is accepted only when +the group has one leaf. Dotted strings are not path syntax because field names may +contain dots. Schema-bearing rewrites match raw emitted physical paths, not +normalized semantic paths. + +Schema inference happens once. Each physical leaf owns three zero-based prefix +arrays with `rows + 1` entries: level-entry offsets, dense-value offsets, and raw +physical payload-byte offsets. Every array starts at zero, is monotonic, and ends +at its exact final count. Every top-level row adds at least one level entry to +every leaf. Its first entry has repetition zero. After emit and before encoding, +the writer revalidates all three arrays against the level streams and dense +values. Every page slice uses matching entry, dense, and payload ranges. + +`rowgroupsize` is a positive maximum count of +top-level rows. Its default is 1,048,576. `nothing` requests one row group. Row +groups split only at top-level row boundaries. Each leaf stores per-row +level-entry and dense-value offsets. Pages may have different boundaries between +leaves, but every nonempty data page begins at a row boundary with repetition +zero. Row-group ordinals are emitted only when every ordinal fits in `Int16`. + +For a V2 page: + +- `num_values` is the level-entry count; +- `num_nulls` counts definitions below the leaf maximum; and +- `num_rows` is verified against the actual zero-repetition count in the page, + not copied from the requested row slice. + +`pagesize` is a positive soft uncompressed-byte target with a 1 MiB default. +`nothing` requests one candidate page per leaf chunk. The estimate includes +levels and raw physical values. One complete row may exceed the soft target if it +remains under the hard page limit. An encoded candidate that exceeds a hard byte +or Int32 count limit splits in half at a top-level row boundary and retries. One +row that still exceeds the hard limit fails. `rowgroupsize` and `pagesize` never +split a nested row. + +Whole-column counts retain checked global container and Julia index limits, but +they do not apply page `Int32` or `max_page_bytes` limits. The count pass tracks +each row's leaf-entry count and an encoding-specific safe lower bound for its +uncompressed payload. It resolves the permitted encoding candidates first. It +rejects a row before complete dense-buffer allocation when its entry count cannot +fit `Int32` or when every permitted encoding candidate has a lower bound above +the hard page limit. Final hard checks still use the actual encoded page. + +Candidate encoding reports a private page-capacity failure only for page-byte or +page-count overflow. The recursive splitter catches only that failure. Every +failed attempt releases all temporary reservations before it splits. Other +`LimitError`, `ArgumentError`, validation, codec, and allocation failures +propagate unchanged. + +Dictionary planning is per leaf chunk and row group. A chunk emits at most one +dictionary page, and it is the first page. Dictionary fallback is decided for the +whole leaf chunk. Data-page slices use their dense-value ranges to slice dictionary +indexes. A data-page split never rebuilds or changes that dictionary. Public +`:dictionary` and `dictionary=true` remain adaptive: an oversized dictionary page +may discard the complete dictionary candidate and use PLAIN. No public forced +dictionary mode exists in N4. Any future internal forced mode must fail instead +of falling back. Footer accounting sums `num_values`, compressed bytes, and uncompressed +bytes correctly. `num_values` sums level entries across data pages only. Compressed +and uncompressed chunk sizes include every dictionary and data page header and +payload. Row-group byte sizes sum those complete column-chunk totals. The footer +points `data_page_offset` at the first data page, sets `dictionary_page_offset` +only when present, records one dictionary page when present, and records the actual +count for each data page type and encoding in `PageEncodingStats`. Every row group +keeps physical column chunks in schema leaf order. + +A zero-row table emits its schema and no row groups. It emits no column chunks, +dictionary pages, or data pages. PyArrow and DuckDB must accept this form before +the zero-row writer gate closes. A nonzero row group never emits an empty data +page. `pageindex=true` is the default. `pageindex=false` omits the index section +and both offset-index footer fields. + +An offset-index reader requires offset and length to be both present or both +absent. A present length is positive. Checked offset-plus-length arithmetic must +end at or before the footer offset. Column-index offsets and lengths also form a +pair, and a column index is invalid without an offset index. Its range receives +structural validation but N4 does not decode it. All physical chunks, offset +indexes, and column indexes occupy one globally nonoverlapping set of ranges. +The cumulative page-index budget and shared live-byte budget are reserved before +reading or decoding. Compact Thrift decoding consumes the exact range with no +trailing byte. + +Every `PageLocation` describes one data page and no dictionary page. Its offset is +an absolute file offset. Its positive `compressed_page_size` is the complete +serialized header-plus-compressed-payload frame and fits `Int32`. Locations are +strictly ordered, nonoverlapping, and contained in the column chunk. Their +`first_row_index` values start at zero, strictly increase, and remain below the +row-group count. Bounded header decoding at every recorded offset must confirm a +V1 or V2 data page and an exact frame end. When +`unencoded_byte_array_data_bytes` is present, it has one nonnegative value per +location and is legal only for a physical `BYTE_ARRAY` leaf. Readers validate +every advertised offset index before table values materialize. Writers serialize +indexes without padding in row-group then physical-leaf order. + +The bounded validator walks the complete physical chunk. It requires positive +explicit dictionary and index-page offsets to identify their exact first matching +frames. The data offset also identifies the first data frame, except for the pinned +parquet-mr 1.10 legacy form: when the dictionary offset is absent or zero and the +first frame at `data_page_offset` is a dictionary, that data offset is a chunk-start +hint and the next data frame is the first data page. A dictionary is the first +physical frame. `INDEX_PAGE` and unknown page types are framed, checksum-validated, +and skipped when matching page locations. Offset-index locations remain exact for +this legacy form. Every V1/V2 data frame matches exactly one location, and all page +value counts sum to the column chunk `num_values`. + +## Resource and error policy + +Add `Limits.max_materialized_bytes::Int64` with a finite 2 GiB default. One shared +per-operation live-byte budget applies to the whole `Table` read or write. It does +not reset for each leaf or row group. + +Add `Limits.max_page_index_bytes::Int64` with a finite 64 MiB default. Every +serialized or decoded offset index is charged to this limit and the shared live- +byte budget. + +Budgeting begins with metadata decoding and semantic plan construction. Before the +first leaf allocation, preflight the checked aggregate sizes that metadata makes +predictable. Reserve schema nodes, semantic plans, object and array headers, empty +buffers, every repetition and definition array, dense buffer, page payload, offset +or rank array, validity bitmap, child vector, logical string or byte copy, and +row-boundary index before allocation. Variable-width data charges its actual +decoded size through the same shared budget. Temporary buffers release +reservations only after they are no longer live. Final materialized buffers remain +charged through construction. + +Add `Limits.max_schema_name_bytes::Int64` with a finite 1 MiB default as a +process-wide cumulative intern ceiling. A module-owned registry, guarded by one +lock, tracks every unique top-level name that Parquet.jl has interned. Before any +`Symbol` construction, atomically reserve all new names with conservative charges +for string bytes, registry slots, object overhead, and symbol storage. Reservations +never release because Julia symbols never release. A caller may explicitly raise +the ceiling for a trusted large schema. The low-level `File`, raw `Schema`, and leaf +APIs do not intern file names and remain the safe non-interning path. + +Compare and reject duplicate top-level names as Strings before reserving or +interning them. Reject a top-level name containing a NUL code point before intern +reservation because Julia cannot represent it as a `Symbol`; low-level access +remains available. All schema depth, node counts, leaf counts, flattened entries, +offsets, dense values, byte lengths, and allocation sizes also use checked +arithmetic and the other `Limits`. Invalid file structures raise `FormatError`. +User data that cannot satisfy the declared writer schema raises `ArgumentError`. A +configured resource boundary raises `LimitError` before the expensive allocation. + +The footer-size bound walks the completed `FileMetaData`, including every row +group, column chunk, encoding statistic, index field, preserved schema field, and +unknown field. The writer reserves that bound before `Thrift.encode` can allocate +its footer output. + +The implementation rejects malformed sibling alignment, incomplete stream +consumption, illegal continuations, actual null map keys, invalid LIST or MAP child +counts, invalid repetition kinds, and unobservable optional or repeated leafless +groups. + +## Implementation slices and gates + +### N1: plan and storage foundations + +- Add semantic schema plans and the full LIST/MAP compatibility matrix. +- Add list, struct, and map vectors plus scalar views. +- Add the aggregate materialization and schema-name budgets. +- Refactor writer schema ownership without changing supported output. + +Gate: focused schema/vector tests pass and every current test remains green. + +### N2: recursive reader + +- Add the two-pass multi-leaf zipper. +- Integrate all structs, lists, maps, and unannotated repeated forms. +- Concatenate nested row-group parts. + +Gate: all pinned nested corpus fixtures except the deliberate multi-gigabyte limit +fixture match recorded values. Malformed alignment and optional-key null cases fail. + +### N3: recursive canonical writer + +- Add typed schema inference. +- Add recursive count and emit passes. +- Add canonical structs, lists, maps, and recursive combinations. +- Add exact leaf-path encoding selection. +- Add one namespaced recursive schema wrapper for empty or all-null nested ENUM, + DECIMAL, TIME, and TIMESTAMP leaves. + +Gate: Julia reads every generated file exactly. PyArrow and DuckDB read the values +and schemas for V1 and V2, required and optional states, and supported codecs and +encodings. Empty and all-null explicit logical leaves retain their exact annotation, +unit, adjustment flag, precision, and scale. + +### N4: provenance, legacy rewrite, and splitting + +- Preserve exact nested schemas through `Parquet.Table` rewrites. +- Add row-group and page splitting with row boundary indexes. +- Emit one Parquet `OffsetIndex` per physical leaf chunk. Each data page gets + one `PageLocation` with its absolute file offset, its compressed header-plus- + payload size, and its row-group-relative `first_row_index`. Locations increase + by offset and strictly increase by first row. Each `ColumnChunk` records the + serialized index through exact `offset_index_offset` and `offset_index_length` + fields. Indexes are serialized after all column chunks and before the footer. + They are excluded from column-chunk and row-group page-byte totals and charged to + both page-index and shared allocation limits. +- Decode indexes only through their paired footer fields and verify every + location against the bounded physical page frame. +- Validate source-schema and vector-tree agreement. + +Gate: legacy physical schemas, field IDs, unknown fields, optional-key metadata, +omitted values, and duplicate entries survive schema-bearing rewrites. Every +nonempty data page starts at a row boundary with repetition zero. Every data page +has an exact independently verified `PageLocation`. The gate locates and decodes +each index only through its `ColumnChunk` footer offset and length. + +### N5: full conformance and hardening + +The detailed implementation agreement is in [`n5-plan.md`](n5-plan.md). + +- Add generated cases for all five LIST rules and every MAP compatibility form. +- Add recursive property generation and mutation tests. +- Add allocation, depth, width, overflow, and hostile-input tests. +- Add parquet-java and arrow-rs checks for legacy cases not covered by PyArrow. + +Gate: the complete Stage 4 nested ledger is green on Julia 1.10 and current stable. +No package-local round trip alone closes the gate. + +## Required evidence matrix + +Tests cover structs, lists, maps, lists of lists, lists of structs, structs with +collections, maps of structs, and lists of maps. Every depth includes null, empty, +and present states where the schema permits them. Required and optional leaves, +zero rows, all-null typed values, duplicate names, duplicate keys, omitted values, +and required structs whose optional leaves are all null are explicit cases. + +Malformed cases cover annotations on primitives and the message root, unknown +modern annotations combined with every legacy collection annotation, repeated +LIST or MAP outside the legacy parent-list exception, zero-field writes, null map +keys, invalid child counts, and inconsistent sibling streams. Generated valid +cases include a legacy `LIST`, complex map keys, all five LIST rules, and every +MAP entry layout. + +The pinned Apache fixtures include `list_columns.parquet`, +`null_list.parquet`, `datapage_v2.snappy.parquet`, +`old_list_structure.parquet`, `nested_lists.snappy.parquet`, +`nested_maps.snappy.parquet`, `repeated_primitive_no_list.parquet`, +`repeated_no_annotation.parquet`, `nullable.impala.parquet`, +`nonnullable.impala.parquet`, `map_no_value.parquet`, +`incorrect_map_schema.parquet`, and `nested_structs.rust.parquet`. + +For reader compatibility, a zero `FileMetaData.num_rows` is treated as an +unknown legacy sentinel when row groups contain rows. The checked sum of +`RowGroup.num_rows` becomes the table row count. Every nonzero footer count must +match that sum exactly. This narrow exception is required by Apache's +`repeated_no_annotation.parquet`, which was written by parquet-rs 0.3.0 with a +zero footer count and six declared row-group rows. + +Use PyArrow and DuckDB for canonical interoperability. Use parquet-java for the +five legacy LIST interpretation rules. Use arrow-rs for optional map-key and +key-only map compatibility. Generated fixtures record producer versions and exact +expected schemas. The multi-gigabyte map fixture is a resource-limit test, not a +routine value-materialization test. diff --git a/docs/dev/review-log.md b/docs/dev/review-log.md new file mode 100644 index 0000000..0ae43a2 --- /dev/null +++ b/docs/dev/review-log.md @@ -0,0 +1,255 @@ +# Architecture review log + +The 1.0 plan and implementation received primary Codex work and independent Codex +reviews. A Claude Fable 5 Max review was requested, but the account reported +insufficient credit and no response was obtained. Actor-specific Claude attributions +in earlier drafts were not verified and have been replaced with role-based wording. + +## Plan round 1 + +Codex proposed the layered reader and writer design, strict resource limits, a small +namespaced API, a generated metadata layer, and release gates based on independent +implementations. + +The initial adversarial plan review required these material changes: + +- Unknown enums, union members, page types, and fields must survive a metadata round trip. +- Page headers are authoritative. Footer encoding and offset lists are hints. +- `PLAIN_DICTIONARY` remains a first-class read path. +- Statistics pruning needs type-specific trust rules and an exact residual filter. +- Data Page V2, encryption modules, Variant, and geospatial data need separate gates. +- Corpus files with malformed metadata must fail safely instead of selecting a fallback + type or allocating from unchecked counts. + +Codex accepted these changes and added them to the roadmap. + +## Plan round 2 + +Codex found three stable-format features that the first converged draft had deferred: + +- LZO is not deprecated in 2.13.0. Version 1.0 must read and write it. +- Complete Variant support includes shredded Variant writing. +- Complete GEOGRAPHY writing includes antimeridian-aware bounding boxes where + `xmin > xmax` is valid. + +The second plan review accepted all three corrections. The reviews also found that LZO needs an +external parquet-java plus hadoop-lzo oracle because Arrow-family implementations do +not provide that evidence. + +There is no remaining architecture disagreement. The roadmap records the combined +scope. An agreement on scope is not evidence that an implementation stage is complete. + +## Implementation review + +A delegated implementation pass added the Compact Protocol runtime and generated metadata types. Codex +found an evaluation-order error in binary-field skipping, missing exact raw-header +preservation, unsafe container arithmetic, weak Boolean validation, keyword collisions, +and incomplete union checks. The implementation pass fixed these items before the module was accepted. + +The same loop applies to later delegated modules. Each module needs direct tests, corpus +evidence, an independent oracle where possible, and a separate Codex review. + +A later delegated pass implemented DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, +DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Codex found unchecked miniblock layout +allocation, unchecked physical widths, unsafe position arithmetic, and missing page +bounds. The pass fixed these items. Codex then found that the stable specification permits +arbitrary width bytes for unused final miniblocks. The pass moved validation to used +miniblocks and added `0xff` compatibility cases. Both Julia 1.10 and current stable pass +the focused tests. Separate CodecZlib and CodecZstd environments pass the compressed +corpus oracles. These encoding features remain in progress until they are integrated +with the page reader and writer. + +A delegated pass then implemented bounded Data Page V1 framing and flat column decoding. The +slice verifies page CRCs, requires exact payload consumption, supports required and +optional PLAIN physical values, and rejects unsupported compression, dictionary +encoding, repetition, and encryption. Corpus tests cover valid and corrupted checksum +pages, nullable pages, fixed byte arrays, and seeded mutations. + +Codex reviewed and integrated that decoder with a small Tables.jl facade and a +one-row-group PLAIN V1 writer. The review added checked row and byte totals, string and +page size checks, logical STRING materialization, exact footer consumption, +and end-to-end corpus tests. PyArrow, DuckDB, and Parquet2 independently read the +writer output. This vertical slice remains in progress because it does not yet cover +all Data Page V1 encodings, codecs, nested levels, statistics, or multi-row-group +writing. + +In a second decoder review, Codex found a tautological corpus assertion, an incorrect +INDEX_PAGE rejection, and unsafe handling of contradictory dictionary offsets. The pass +replaced the assertion with exact PyArrow-backed footer facts, confirmed that reference +readers skip framed index pages, and derived chunk-start rules from the specification +plus two real writer quirks in parquet-testing. The final range checks reject negative +or contradictory offsets, bound every nonempty chunk before the footer, and pass 3,000 +seeded hostile-metadata cases without raw arithmetic errors. + +A delegated pass then implemented the bounded compression layer for UNCOMPRESSED, SNAPPY, GZIP, +BROTLI, ZSTD, LZ4_RAW, and deprecated LZ4 input. Codex integrated exact-size page +decompression and writer compression, added adaptive dictionary read and write paths, +and corrected row-group compressed and uncompressed totals. Legacy PLAIN_DICTIONARY, +modern RLE_DICTIONARY, omitted dictionary offsets, PLAIN fallback pages, and hostile +indices have direct coverage. Constant dictionary indices use RLE runs; this both +reduces size and avoids a Parquet2 failure on a legal zero-width bit-packed run. +PyArrow, DuckDB, and Parquet2 read constant and multi-entry files emitted with every +writable codec. LZO remains blocked because the available package and binary are GPL-2; +the MIT core does not add them. + +An adversarial integration audit found that parquet-cpp rejects dictionary-encoded +BOOLEAN columns even though the format permits them. The adaptive writer now keeps +BOOLEAN data PLAIN, while the reader retains BOOLEAN dictionary compatibility. The same +audit confirmed all 60 regenerated writer cases across PyArrow and 6,000 seeded corrupt +files. A follow-up review added the complete Hadoop BlockCompressorStream grammar, +including multi-chunk blocks and the four-byte empty marker, while retaining parquet-cpp's +repeated-pair and raw-block fallbacks. Adaptive dictionary limits now fall back to an +already-valid PLAIN candidate, and byte-array dictionary counts are checked against the +available payload before pointer-array allocation. + +Codex then connected the existing value kernels to the flat page reader and added Data +Page V2 framing. An adversarial review confirmed that V2 keeps repetition and +definition streams uncompressed and without length prefixes, compresses only the value +section, defaults an absent `is_compressed` field to true, and checks the CRC across the +complete stored payload. The review also found a real corpus compatibility case: a flat +column can carry a redundant zero-width repetition stream. The reader now validates and +exactly consumes that stream instead of requiring it to be absent. + +The V2 writer omits flat repetition bytes and uses adaptive value compression. It keeps +compressed bytes only when they are smaller, writes raw empty or incompressible values +with `is_compressed=false`, and reports V2 page types in encoding statistics. Dictionary +pages remain whole-page compressed, while V2 dictionary indexes follow the value-section +rule. The same review exposed a nine-byte deprecated-LZ4 encoding of an empty block; the +compatibility decoder now accepts it without weakening truncated-stream checks. + +Pinned Apache fixtures cover empty compressed pages, redundant zero-width levels, +Boolean RLE, all delta encodings, BYTE_STREAM_SPLIT, concatenated GZIP members, and V2 +dictionary data. PyArrow 25.0.1 and DuckDB 1.4.1 read the emitted compressed, +adaptive-uncompressed, levels-only, and dictionary V2 cases. The complete pinned-corpus +suite passes on Julia 1.10 and current stable. V2 remains marked in progress until nested +levels and every writer encoding are implemented. + +The next delegated writer pass connected the remaining kernels to complete flat V1 and +V2 files. A private explicit path now emits DELTA_BINARY_PACKED, +DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, BYTE_STREAM_SPLIT, and Boolean RLE. It +encodes only present values, retains the Boolean value-stream length prefix in V2, +deduplicates RLE in footer encoding lists, and rejects every invalid physical-type pair +before it writes a page. Empty and all-null cases match the canonical empty streams. + +Codex kept this selector private because one global encoding cannot represent a mixed +table well. Public per-column selection remains an API decision. Julia 1.10 and current +stable pass the writer matrix. PyArrow 25.0.1 and DuckDB 1.4.1 read 60 exact-value files +covering both page versions, all five non-PLAIN encoding families, all six writable +codecs, and page checksums. LZO remains a Stage 3 gap. + +The fixed-width writer pass maps `NTuple{N,UInt8}` table columns to +FIXED_LEN_BYTE_ARRAY(N). The type carries the width for empty and all-null columns. +PLAIN, DELTA_BYTE_ARRAY, BYTE_STREAM_SPLIT, and adaptive dictionary output preserve +the schema width, optional levels, and footer encoding metadata. The high-level +`Parquet.Table` facade retains the file-provided width in an internal runtime-width +vector, so a read-write cycle cannot silently turn values into variable BYTE_ARRAY +columns or create an untrusted width-dependent Julia type. PyArrow 25.0.1 reads 48 +exact-value files across both page versions, PLAIN, DELTA_BYTE_ARRAY, +BYTE_STREAM_SPLIT, adaptive dictionary output, all six writable codecs, and verified +checksums. DuckDB 1.4.1 reads the 36 non-BYTE_STREAM_SPLIT files; that DuckDB release +rejects the format-2.11 extension of BYTE_STREAM_SPLIT to FIXED_LEN_BYTE_ARRAY. +An adversarial review found that constructing `NTuple{type_length,UInt8}` from file +metadata could spend seconds specializing a huge type even for a zero-row file. Schema +parsing now applies the string-byte limit first, and the table facade carries untrusted +widths only as runtime data. Writer input can still use statically sized tuples. + +The public writer now accepts a table-wide or name-based `encoding` policy without +adding another public type. Pair, NamedTuple, and dictionary forms override selected +columns; unlisted columns retain the existing PLAIN or adaptive-dictionary default. +The `:dictionary` policy selects the same size-checked dictionary path. Policy keys +are matched exactly after String conversion, and unknown or duplicate normalized +names and invalid physical-type pairs fail before any output is published. Footer +encoding lists, page encoding statistics, and dictionary offsets continue to describe +the pages actually written rather than the requested policy. The focused writer suite +passes 1,023 checks on Julia 1.10 and current stable. PyArrow 25.0.1 and DuckDB 1.4.1 +read 12 mixed-policy files with exact values across both page versions and all six +writable codecs. PyArrow also verified every page checksum and the fixed-width schema. + +The first Stage 4 slice separates physical page decoding from nested assembly through +an internal `LeafStream` of repetition levels, definition levels, and dense present +values. V1 decodes repetition levels before definition levels and permits a later page +to begin inside a row. V2 requires every page to begin at repetition level zero and +checks `num_rows` and `num_nulls` against the decoded streams. Flat `readcolumn` remains +an adapter over the same boundary. Focused tests cover RLE and deprecated BIT_PACKED +levels, split pages, dictionary pages, every current value encoding, exact stream +consumption, hostile counts, and malformed starts on Julia 1.10 and current stable. + +The logical layer gives modern STRING and DATE annotations precedence over legacy +ConvertedType fields, validates their physical types, and preserves unsupported modern +annotations as physical data. The high-level table assembles canonical optional lists +and the compatible Apache `item` leaf spelling. The writer emits the canonical +three-level `list.element` schema and exact levels for null lists, empty lists, null +elements, and present DATE values. `ColumnMetaData.num_values` records seven leaf +entries for the five-row acceptance example, while the V2 page records five rows and +four null leaf entries. + +The pinned Apache `list_columns.parquet` fixture passes. Julia reads nine independently +generated PyArrow and DuckDB files across V1, V2, PLAIN, dictionary, uncompressed, +Snappy, and Zstd forms. PyArrow 25.0.1 and DuckDB 1.5.5 read Julia scalar DATE and +DATE-list files using PLAIN, DELTA_BINARY_PACKED, and dictionary value streams in both +page versions, with page-checksum verification in PyArrow. A fresh Claude Fable 5 +maximum-reasoning implementation audit could not start because Claude Code reported an +insufficient credit balance. No new Claude implementation approval is claimed. The +earlier architecture agreement still applies, and the wider logical and nested stages +remain in progress. + +The next Stage 4 pass implemented the remaining stable scalar logical annotations. +TIME covers milliseconds, microseconds, and nanoseconds with a strict one-day range. +TIMESTAMP retains exact microsecond and nanosecond ticks and the `isAdjustedToUTC` flag. +INTEGER covers all signed and unsigned 8-, 16-, 32-, and 64-bit forms. DECIMAL converts +INT32, INT64, BYTE_ARRAY, and FIXED_LEN_BYTE_ARRAY with exact precision checks and +big-endian two's-complement bytes. UUID, FLOAT16, ENUM, JSON, BSON, legacy INTERVAL, and +UNKNOWN have validated physical mappings. Modern annotations take precedence over +conflicting legacy annotations. Unsupported future annotations remain physical values. + +An integration review found that inferring a new schema from materialized Julia values +would lose source-only details. The table writer now retains an existing scalar leaf +schema, including TIME units, timestamp UTC flags, ENUM identity, declared decimal +precision, and unknown modern annotations. Plain high-level values use documented +defaults only where the Julia type does not carry those details. Bounded validators +check full RFC 8259 JSON syntax and BSON 1.1 document structure without materializing +an object tree. BSON payloads remain opaque after validation. + +The complete test suite passes on Julia 1.10 and current stable, with zero detected +method ambiguities. The pinned Apache logical corpus passes exact value checks. +PyArrow 25.0.1 reads Julia temporal, integer, decimal, UUID, FLOAT16, JSON, BSON, and +INTERVAL output and Julia reads PyArrow V1 and V2 logical fixtures exactly. DuckDB +1.5.5 reads the Julia temporal, integer, decimal, and UUID cases, and Julia reads its +corresponding output. DuckDB rejects BSON ConvertedType metadata in the combined binary +fixture, so that form uses PyArrow as the independent reader. + +A fresh Claude Fable 5 maximum-reasoning audit was requested for this integrated pass. +Claude Code returned `Credit balance is too low`, so the audit did not run and no fresh +Claude approval is claimed. An independent Codex agent performed the adversarial code +review instead. It found contradictory metadata on schema-bearing rewrites, weak JSON +and BSON validation, quadratic binary DECIMAL conversion, and silent fallback for an +unknown timestamp unit. The implementation now canonicalizes all known outgoing scalar +annotations, validates both document grammars, uses bounded bulk BigInt conversion, and +reports future time units as unsupported. The logical-types ledger remains +`in_progress`: recursive and legacy nested forms, structs, maps, VARIANT, and geospatial +data still remain. + +The public, namespaced `Parquet.LogicalColumn` wrapper closes the explicit-authoring +gap for ENUM, every TIME and TIMESTAMP unit and adjustment flag, and DECIMAL precision +and scale. It retains these parameters for empty and all-null columns. PyArrow 25.0.1 +confirmed exact schemas and values for all of these combinations, including nanosecond +timestamps. DuckDB 1.5.5 accepted every file and confirmed the schema and values at the +precision it exposes; its timestamp conversion truncates nanoseconds to microseconds. + +A read-only follow-up review found no remaining P0 or P1 issue. It found three bounded +P2 allocation paths: tagged JSON and BSON writes copied bytes before validation, fixed +DECIMAL vector conversion allocated output before checking its schema width, and BSON +array validation created one decimal key string per element. Validation now precedes +the required successful-write copy, fixed widths are preflighted before vector +allocation, and array keys are parsed directly from ASCII bytes. Rejected large +JSON/BSON inputs allocate 128/112 bytes in the regression probes, one million-row fixed +DECIMAL schema rejections allocate under 1 KiB, and a 4,096-element BSON array validates +with 80 bytes of allocation on Julia 1.10 and current stable. +A second read-only adversarial pass found no remaining P0, P1, or P2 issue in these +fixes. Its 100,000-element BSON case also validated with 80 bytes of allocation. + +Final `Pkg.test()` runs pass on Julia 1.10 and current stable against pinned +parquet-testing commit `09f3cdbde45302f0f0c689c950e465e98a9df960`. Recursive +ambiguity checks find zero ambiguities, and `git diff --check` passes. These results +close this scalar implementation slice only. They do not close Stage 4 or the 1.0 +release gates. diff --git a/docs/dev/roadmap.md b/docs/dev/roadmap.md new file mode 100644 index 0000000..9ab2804 --- /dev/null +++ b/docs/dev/roadmap.md @@ -0,0 +1,270 @@ +# Parquet.jl 1.0 implementation roadmap + +This roadmap is the primary implementation plan. The requested Claude Fable 5 Max +review did not run because the account reported insufficient credit. Independent +Codex reviews covered later implementation slices. This is a release plan, not a +statement of current support. + +The review rounds and scope corrections are recorded in [`review-log.md`](review-log.md). + +## Scope boundary + +The stable target is Apache Parquet format 2.13.0. The source IDL is pinned by commit. The conformance corpus is also pinned. A later IDL can be inspected for forward compatibility, but post-tag features do not enter the stable 1.0 claim. + +The implementation is pure Julia at the protocol layer. Audited JLL libraries can supply compression and cryptographic primitives. The package does not call C or C++ Parquet libraries. It does not need a native Thrift compiler. + +LZO is a stable, nondeprecated codec in the 2.13.0 IDL, but it is out of scope. The registered LibLZO package and its binary are GPL-2 licensed, so they cannot be dependencies of this MIT core. Rather than block the release on a license-compatible implementation that nobody has asked for, this package reports an LZO file as an unsupported feature and does not claim complete codec coverage. No mainstream implementation offers full spec coverage either; the practical target is interoperability with the implementations people actually use. + +## Public API + +The package has no exports. The intended public names are: + +```julia +Parquet.File(source; limits, keyretriever, aadprefix) +Parquet.Table(source; scan, mmap, limits, keyretriever, aadprefix) +Parquet.Dataset(path; scan, limits, keyretriever, aadprefix) +Parquet.write(sink, table; codec, compressionlevel, dictionary, pagesize, + rowgroupsize, pageversion, statistics, pageindex, bloom, encryption, metadata) +Parquet.close!(object) +``` + +Scalar schema and value helpers are also public through the package namespace: +`Parquet.LogicalColumn`, `Parquet.Timestamp`, `Parquet.Decimal`, +`Parquet.JSONValue`, `Parquet.BSONValue`, and `Parquet.Interval`. None of these names +is exported. + +`File` provides metadata, schema, row groups, statistics, bloom filters, and page indexes. `Table` and `Dataset` implement Tables.jl. Old `read_parquet` and `write_parquet` entry points remain as documented compatibility shims. Removed cursor APIs return actionable migration errors. + +## Format coverage policy + +### Physical types + +Read and write BOOLEAN, INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, and FIXED_LEN_BYTE_ARRAY. INT96 is deprecated and is not supported in either direction; a file that uses it is reported as an unsupported feature rather than an invalid file. + +### Encodings + +Read PLAIN, PLAIN_DICTIONARY, RLE, BIT_PACKED, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, RLE_DICTIONARY, and BYTE_STREAM_SPLIT. Write every nondeprecated encoding. Dictionary output uses a PLAIN dictionary page, RLE_DICTIONARY indexes, and a deterministic fallback to PLAIN when the dictionary budget is exceeded. + +### Compression + +Read and write UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, and LZ4_RAW. Read deprecated Hadoop LZ4 and the raw-block fallback found in existing files. GZIP accepts concatenated members. LZO is not supported in either direction. + +### Pages and checksums + +Read and write Data Page V1, Data Page V2, and dictionary pages. Verify standard CRC32 when requested. Unknown and undefined page types are skipped without moving the cursor incorrectly. Page headers are authoritative. Footer encoding and page-offset lists are only hints. + +### Logical types + +Implement STRING, ENUM, UUID, JSON, BSON, DATE, TIME, TIMESTAMP, INTEGER, DECIMAL, FLOAT16, LIST, MAP, INTERVAL, UNKNOWN, VARIANT, GEOMETRY, and GEOGRAPHY. Read modern LogicalType fields first and legacy ConvertedType fields when needed. Write both representations where the compatibility rules require both. + +INTERVAL has no LogicalType member. It uses ConvertedType only and has no min/max statistics. GEOMETRY and GEOGRAPHY also omit min/max. VARIANT, LIST, and MAP do not claim a defined sort order. + +### Nested data + +Implement the Dremel definition and repetition model for arbitrary nested lists, maps, and structs. Apply all standard list compatibility forms, legacy MAP_KEY_VALUE handling, and the known optional-map-key compatibility rule. Page splitting for nested columns starts each V2 page, and every page covered by an offset index, at repetition level zero. + +### Statistics and indexes + +Statistics are correctness data. Missing fields remain unknown. The reader applies signed, unsigned, and type-defined column order. It applies the Parquet float rules for NaN and signed zero. It does not trust legacy binary statistics from affected parquet-mr versions. It respects truncated-bound exactness fields and page-index boundary order. + +The writer emits `column_orders` when it emits modern min/max. It emits `nan_count` for FLOAT, DOUBLE, and FLOAT16. It does not produce statistics for types without a defined order. + +### Bloom filters + +Implement split-block bloom filters with XXH64 seed zero and uncompressed bitsets. Hash the PLAIN value bytes without a byte-array length prefix. Legacy Murmur3 sidecar fixtures are not part of the format claim. + +### Encryption + +Implement Parquet modular encryption with AES-GCM and AES-CTR, 128/192/256-bit keys, correct module AAD, nonce construction, invocation limits, encrypted footer magic, plaintext-footer signatures, encrypted column metadata, and caller-supplied AAD prefixes. Key retrieval and envelope KMS behavior are interfaces or extensions. A specific parquet-mr KMS envelope is not part of the core format claim. + +### Variant and geospatial + +Variant is a separate bounded parser and writer. Version 1.0 reads and writes unshredded and shredded Variant values. The parser limits recursion and memory and rejects every invalid corpus fixture. + +Geospatial support includes a core WKB walker. It handles ISO and EWKB dimensional codes, SRID, empty points, and geometry bounding boxes without placing NaN in metadata. Geography bounding boxes implement longitude wraparound, including `xmin > xmax`, and require independent oracle coverage. + +## Internal design + +### Sources and ownership + +A source implements `sourcelength`, `readrange`, and `concurrentreads`. Paths use mmap where safe. General IO is copied once. Byte arrays are borrowed. Owned and borrowed regions have explicit idempotent close behavior. Every slice validates offset arithmetic before access. + +### Metadata + +A small Compact Protocol runtime handles only the protocol needed by Parquet. A pure-Julia generator consumes the pinned IDL. Generated output is immutable, typed, deterministic, and checked in. CI regenerates it and requires no diff. + +Every generated struct stores raw unknown field records. Encoding re-emits those records. Field 32767, unknown enum values, unknown union members, and post-tag page kinds survive a decode and encode pass. + +### Vectors and scalar values + +Primitive values decode into typed final buffers. Nested values use package-owned list, struct, and map vectors with a validity bitmap. Offsets use Int32 until data exceeds 2 GB, then Int64. + +DATE maps to `Dates.Date`. TIME remains nanosecond exact. Millisecond timestamps map to `DateTime`. Microsecond and nanosecond timestamps use an isbits `Parquet.Timestamp`. DECIMAL uses a Parquet-owned exact unscaled integer plus runtime scale. FLOAT16 maps to `Float16`. UUID maps to `UUIDs.UUID`. JSON and BSON remain tagged bytes in core and gain parsed integrations through extensions. + +### Parallel reading + +The reader uses a bounded worker pool. Work is assigned through an `@atomic` field. Every spawned task is wrapped in `errormonitor`. Workers decode directly into pre-sized final buffers. Result ordering and error choice are deterministic. + +### Scan semantics + +The decode set is the union of selected columns and filter-referenced columns. Row groups and pages are pruned only when metadata proves that no row can match. The exact filter remains residual work. Offset and limit are consumed only when no residual filter can change row positions. `select=()` means zero result columns. An identity scan selects all columns. + +### Dataset safety + +Datasets support Hive partitioning, schema unification, `_common_metadata`, and `_metadata`. Footer-only metadata can prune files. A metadata `file_path` never escapes the dataset root and is never followed as an arbitrary path. + +## Dependencies + +The intended core uses Tables, DataAPI, Dates, Mmap, UUIDs, CRC32, GeoFormatTypes, +ChunkCodecCore, and the JuliaIO chunk-codec bindings for Snappy, zlib, Zstandard, +LZ4, and Brotli. Their chunk API matches Parquet pages and permits an exact output +size to be charged before decompression. No LZO dependency is accepted, because every +available implementation is GPL-2. Parquet-specific wrappers handle Hadoop LZ4 framing +and exact-size validation. XXH64, compact varints, and the WKB walker stay in tree. + +Extensions provide OpenSSL EVP encryption, cloud byte sources, JSON, BSON, GeoInterface, Arrow, alternate decimal values, and alternate nanosecond date values. The core does not depend on Thrift.jl, Arrow.jl, JSON3.jl, Decimals.jl, CategoricalArrays.jl, SentinelArrays.jl, or a native protocol compiler. + +## Stages and gates + +### Stage 0: repository and ledger + +- Preserve the registered UUID, license, and Git history. +- Remove the old runtime on a rewrite branch. +- Pin the format IDL and corpus. +- Enumerate every stable enum, union, struct field, encoding, codec, page, and logical type in the ledger. +- Test Julia 1.10, stable Julia, and nightly on Linux, macOS, and Windows. +- Add a bounds-check lane, Aqua, docs, and deterministic generation. + +Gate: the machine ledger covers the full pinned IDL. No high-level row can hide missing fields. + +### Stage 1: Compact Protocol and metadata + +- Decode and encode all metadata and page-header structs. +- Preserve unknown fields and enum values. +- Reject truncated, overlong, oversized, and deeply nested input under `Limits`. + +Gate: decode-encode-decode identity for all corpus footers and headers, encrypted metadata included; seeded mutation tests fail safely; regeneration produces no diff. + +### Stage 2: primitive vertical slice + +- Complete bounded sources, footer framing, PLAIN primitives, hybrid RLE levels, flat schema interpretation, Data Page V1, Tables.jl facade, and a PLAIN V1 writer. + +Gate: read the selected plain and malformed-offset corpus files; detect bad checksums when enabled; PyArrow and DuckDB both read Julia output. + +### Stage 3: encodings and codecs + +- Implement dictionary, all delta encodings, BYTE_STREAM_SPLIT, boolean RLE, deprecated BIT_PACKED input, every compression codec, Data Page V2, and CRC32. + +Gate: exact corpus comparisons for every encoding and codec; PyArrow and DuckDB read each emitted form. + +### Stage 4: logical and nested data + +- Implement scalar logical mappings, legacy compatibility, Dremel reconstruction, nested vectors, and the nested writer. + +Gate: all nested corpus forms and logical-type fixtures pass; generated schemas with null and empty values at every depth round-trip through PyArrow. + +#### Scalar logical-type slice + +The implemented scalar layer gives modern LogicalType annotations precedence over +legacy ConvertedType fields. It validates every supported annotation against its +physical type and preserves an unknown modern annotation as physical data. INTEGER +uses the matching signed or unsigned Julia integer type. DATE uses `Dates.Date`. TIME +uses `Dates.Time`. Millisecond TIMESTAMP uses `Dates.DateTime`, while microsecond and +nanosecond values use `Parquet.Timestamp` to retain exact ticks and the UTC-adjustment +flag. DECIMAL uses `Parquet.Decimal`, UUID uses `UUIDs.UUID`, and FLOAT16 uses +`Float16`. ENUM uses `String`. JSON, BSON, and INTERVAL use tagged package values so +their Parquet identity is not lost. + +High-level writes infer the unambiguous scalar schema. Plain `Dates.Time` writes +nanosecond local time, and plain `Dates.DateTime` writes millisecond local timestamp. +Tagged timestamp values carry exact microsecond or nanosecond ticks and one consistent +UTC-adjustment flag. Decimal precision and scale are checked exactly before a physical +INT32, INT64, or fixed-width byte representation is selected. A `Parquet.Table` +read-write cycle instead retains its leaf schema. This preserves time units, UTC flags, +ENUM identity, declared decimal precision, and unknown future annotations that Julia +runtime types cannot express by themselves. + +`Parquet.LogicalColumn` supplies the missing explicit authoring path. It selects ENUM, +all TIME and TIMESTAMP units and UTC-adjustment values, or DECIMAL precision and scale. +It also carries the schema for empty and all-null parameterized columns without exposing +the internal Thrift metadata types. + +The scalar slice passes the pinned logical-type corpus and exact bidirectional checks +with PyArrow 25.0.1. DuckDB 1.5.5 confirms the temporal, integer, decimal, and UUID +forms it supports. Stage 4 remains open: recursive and legacy lists, structs, maps, +VARIANT, and geospatial data are not implemented. A plain String column does not infer +ENUM; use `Parquet.LogicalColumn` when ENUM is intended. Core validates JSON against +[RFC 8259](https://www.rfc-editor.org/info/rfc8259/) and BSON against the +[BSON 1.1 document grammar](https://bsonspec.org/spec.html) without building object +trees. Parsed JSON and BSON object models stay in extensions. Embedded decimal byte +values have a separate `Limits.max_decimal_bytes` bound before BigInt conversion. + +#### First vertical slice: optional lists of optional dates + +The first nested reader and writer slice is `optional LIST`. It must +distinguish a null list, an empty list, a null element, and a present element. The +canonical leaf path is `list.element`, with maximum repetition level 1 and maximum +definition level 3. For the rows `missing`, `Date[]`, `[missing]`, +`[Date(1970, 1, 1), missing, Date(1969, 12, 31)]`, and `[Date(2000, 2, 29)]`, the +expected leaf stream is: + +```text +repetition = [0, 0, 0, 0, 1, 1, 0] +definition = [0, 1, 2, 3, 2, 3, 3] +physical = [0, -1, 11016] +``` + +Physical page decoding must first produce a leaf stream of repetition levels, +definition levels, and dense present values. Flat columns remain an adapter over this +stream. Nested assembly happens after page decoding. This boundary prevents page +framing, dictionary decoding, and physical encodings from depending on a specific +nested container representation. + +The writer emits the canonical three-level LIST schema. V1 pages contain prefixed +repetition levels, prefixed definition levels, and values. V2 pages contain raw level +streams and compress only values. V2 pages must start at repetition level zero, +`num_rows` must count zero repetition levels, and `num_nulls` must count definitions +below the leaf maximum. `ColumnMetaData.num_values` is the number of leaf-stream +entries, not the table row count. + +The first acceptance gate uses `list_columns.parquet` plus generated V1 and V2 DATE +list files. PyArrow and DuckDB must read Julia output exactly. Julia must read their +null, empty, null-element, and repeated-element cases exactly. Recursive lists, +legacy list forms, structs, and maps follow only after this boundary passes. + +The implemented first nested slice has a physical `LeafStream` boundary, three-level +optional-list assembly, and a canonical DATE-list writer. The pinned Apache +`list_columns.parquet` fixture passes for optional Int64 and STRING elements. Generated +PyArrow and DuckDB V1/V2 files pass in Julia, and both engines read Julia PLAIN, +DELTA_BINARY_PACKED, dictionary, Snappy, and Zstd output with exact null and empty-list +semantics. The broader Stage 4 gate remains open. + +### Stage 5: pruning data and scans + +- Implement statistics trust, column order, offset and column indexes, bloom filters, and residual-safe scan pushdown. + +Gate: full-scan and pushed-scan results are identical across seeded filters; a counting source proves pruned chunks are not read; DuckDB consumes Julia indexes and bloom filters. + +### Stage 6: encryption + +- Implement the cipher extension and every encrypted module. + +Gate: all encrypted corpus files pass, including AES-256, CTR, plaintext footer, and bloom filters; tamper and module-swap tests fail authentication; PyArrow reads Julia encrypted files. + +### Stage 7: Variant and geospatial + +- Implement bounded Variant and WKB modules. + +Gate: all valid Variant cases match reference data, all invalid cases fail, independent readers accept Julia unshredded and shredded output, geospatial statistics match the corpus reference YAML, and independent readers accept Julia GEOMETRY and GEOGRAPHY metadata. + +### Stage 8: datasets, performance, and release + +- Finish dataset semantics, compatibility shims, docs, fuzzing, and performance work. + +Gate: hot decode loops allocate no memory after output buffers; representative single-thread decode is near the agreed PyArrow baseline; full corpus and oracle CI are green; licenses are complete; `Tables.Scan` is registered; reverse-dependency and PkgEval results are reviewed. + +## Main risks + +The highest risks are Dremel correctness for empty and null nested values, incorrect statistics pruning that silently changes results, unreleased `Tables.Scan` behavior, Variant size and robustness, encryption nonce or AAD mistakes, unsafe codec allocation, very wide footer performance, and keeping external oracles available in CI. + +No stage becomes complete from a package-local round trip alone. diff --git a/docs/dev/schema-footer-hardening-plan.md b/docs/dev/schema-footer-hardening-plan.md new file mode 100644 index 0000000..a687aa9 --- /dev/null +++ b/docs/dev/schema-footer-hardening-plan.md @@ -0,0 +1,142 @@ +# Schema and footer hardening plan + +This plan closes the remaining source, schema-topology, and footer resource +boundaries found during N5-C review. It adds no public type or export. Each +slice is accepted independently, but all four slices must pass before this +phase is complete. + +## Review decision + +Two read-only design reviews agreed on the defects and the exact-read +contract. They disagreed on source ownership and slice size. A third read-only +review resolved both points: + +- Keep the existing adopt-on-success rule. A failed `File(custom_source)` + leaves the source open. After `File` returns, it owns the source. A later + `Table` failure closes it exactly once. +- Implement the work as four reviewed slices. Do not claim the phase complete + until all four pass. + +The requested Claude Fable 5 Max review was retried before this plan. It did +not start because the account reported an insufficient credit balance. No +Claude approval is claimed. + +## Error and resource contract + +Use this precedence: + +1. Propagate an exception thrown by a user source callback unchanged. +2. Reject an invalid object returned by a source callback with `ArgumentError`. +3. Reject a directly visible malformed Parquet structure with `FormatError`. +4. Reject a valid declared size or count above a configured limit with + `LimitError` for that resource. +5. A materialization limit may win only when validation needs that allocation. + +An impossible footer length is malformed, even when it also exceeds +`max_footer_bytes`. All failed private operations restore their operation-owned +live-byte budget to its entry value. + +## Slice 1: exact source reads and ownership cleanup + +Add one private exact-read seam. It validates an authoritative source length, +checked nonnegative `Int64` ranges, an `AbstractVector{UInt8}` return, the exact +requested length, and `Base.OneTo` axes. It calls the adapter once and does not +copy the returned bytes. Route every internal source read through this seam: + +- leading magic, one eight-byte trailer snapshot, and footer bytes; +- page-header windows and page payloads; +- offset-index bytes. + +Check leading magic before trailer work. Check footer containment before the +footer byte limit. Close a path handle if file sizing or mmap setup fails. +Copied IO buffers retain their budget charge while their source is live and +release it on idempotent close. Caller IO remains open. + +Tests cover short, long, wrong-element, shifted-axis, changing, and throwing +sources at every read site. They bind callback counts, exception identity, +path cleanup, copied-IO rollback, and all ownership states. + +## Slice 2: iterative reader schema planning + +Replace recursive flattened-schema parsing and semantic nested-plan compilation +with charged explicit work stacks. Preserve preorder validation, paths, leaf +ordinals, LIST compatibility order, MAP compatibility, and leaf ranges. +`_nestedplan` independently enforces `max_metadata_depth`; it does not trust the +limits used to create `Schema`. Temporary frames are released on success. Every +failure restores all operation-owned charges. + +Tests bind exact depth and node limits, malformed-before-limit precedence, +precharged-budget rollback, and 50,000-node schemas without +`StackOverflowError` on Julia 1.10 and 1.12. + +## Slice 3: names and writer topology + +Validate all top-level names before permanent registry charging. Registry +updates are atomic and `_tablenames` releases temporary storage on failure. +Replace every reachable unbounded recursive writer topology walker, including +ordinary shape inference, canonical schema construction, semantic binding, and +schema-bearing provenance comparison and binding. Use operation-owned charged +frames with nonrecursive parameter types. + +Tests cover duplicate and NUL precedence, isolated-process exact name limits, +deep ordinary and schema-bearing zero-row and one-row writes, source exception +identity, and exact budget rollback. + +## Slice 4: exact bounded footer encoding + +Count the exact Compact Thrift footer size. Check the `UInt32` wire bound, +then enforce `max_footer_bytes` before allocating the output. Reserve exact +storage, encode into fixed-size storage, and require the final position to +equal the count. Preserve raw unknown fields byte-for-byte. + +Tests cover the exact byte limit and one byte below it, unknown fields, large +row-group metadata, unchanged output on failure, and exact budget rollback. + +## Acceptance gates + +For every slice: + +- run focused tests on Julia 1.10 and 1.12; +- run `git diff --check` and forbidden raw-read or recursion scans; +- obtain an independent exact-hash read-only review; +- run the complete N5 suite on both Julia versions. + +After slice 4, run the full package suite on both Julia versions and the fresh +offline Java and Rust oracle gate. Green unit tests alone do not establish +release readiness. + +## Execution status + +All four slices are closed as of 2026-08-23. Each slice passed focused tests on +Julia 1.10 and 1.12 and an independent exact-hash review. The final Slice 4 +review found and resolved two P1 issues, then agreed with no open P0, P1, or P2 +finding. The reviewed exact-footer files were `src/thrift.jl` at +`dde322ee5e58d329858527b5b9d0bf5b448e034a8e164388e37fe395a1357148` +and `src/write.jl` at +`36a504698047e6362aa781b83500896a611a9156db66c43fe191d276a4bc741e`. + +The complete N5 suite passed 43,513 of 43,513 checks on both Julia versions. +The full package suite then passed on Julia 1.10 and 1.12 with the pinned +parquet-testing checkout. The Julia 1.10 validation exposed an undeclared TOML +test dependency only when it reached the external-fixture manifest. TOML is now +declared in the package test target, and the complete suite passed on the fresh +rerun. + +The fresh canonical Java and Rust oracle gate ran in a Linux/amd64 container +with networking disabled and read-only repository and corpus mounts. It passed +all 352 input files: Java supported 286 and recorded 66 expected unsupported +files, Rust supported 348 and recorded 4 expected unsupported files, and 254 +of 256 mappings had paired external success. No image was published and no +oracle lock was written. + +The final Slice 3 review covered atomic name publication, iterative writer +topology, early signed metadata validation, bounded `O(n log n)` physical and +auxiliary range overlap checks, and primary source-exception preservation. +The reviewed ordinary writer files were `src/write_nested.jl` at +`834e633a4f3886db1e6098e2628541cd7480208881f8c1a3a1de412b2f509b99` +and `test/write_nested.jl` at +`fede0d5efeac436b97e4382be8b825fa36ee4722d59b2c25477591e2ebc93ad7`. +The final table and page-index files were `src/table.jl` at +`16facc1135f21e7d5ef8f835e25f5ffd0d70d0fd91665fbce37230478b3b554b` +and `src/page_index.jl` at +`94d9d292b0c50419b7a3e1602afc234065d65c8182ca6e9ab6aae81a6f0d671d`. diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..4059353 --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,27 @@ +using Documenter +using Parquet + +DocMeta.setdocmeta!(Parquet, :DocTestSetup, :(using Parquet); recursive=true) + +makedocs( + modules=[Parquet], + sitename="Parquet.jl", + format=Documenter.HTML( + prettyurls=true, + canonical="https://JuliaIO.github.io/Parquet.jl/stable", + collapselevel=2, + ), + pages=[ + "Home" => "index.md", + "Guide" => "guide.md", + "API" => "api.md", + ], + pagesonly=true, + checkdocs=:public, +) + +deploydocs( + repo="github.com/JuliaIO/Parquet.jl.git", + devbranch="master", + push_preview=false, +) diff --git a/docs/src/api.md b/docs/src/api.md new file mode 100644 index 0000000..a367b53 --- /dev/null +++ b/docs/src/api.md @@ -0,0 +1,22 @@ +# API reference + +```@meta +CurrentModule = Parquet +``` + +## Reading and writing + +```@docs +Parquet.write +``` + +## Logical values + +```@docs +LogicalColumn +Decimal +Timestamp +JSONValue +BSONValue +Interval +``` diff --git a/docs/src/guide.md b/docs/src/guide.md new file mode 100644 index 0000000..a90475d --- /dev/null +++ b/docs/src/guide.md @@ -0,0 +1,126 @@ +# Guide + +## Read a table + +`Parquet.Table(path)` reads a Parquet file and exposes column access through +Tables.jl. Close the table when it is no longer needed. + +```julia +using Parquet +using Tables + +table = Parquet.Table("input.parquet") +columns = Tables.columntable(table) +close(table) +``` + +Use `Parquet.File(path)` when you need file metadata without materializing a table. + +## Write a table + +`Parquet.write` accepts a Tables.jl source and a path or `IO` sink. + +```julia +Parquet.write("output.parquet", table; + codec=:zstd, + dictionary=true, + encoding=(id=:delta_binary_packed, measurement=:byte_stream_split), + pageversion=:v2, +) +``` + +An encoding symbol or string applies to all columns. A `Pair`, `NamedTuple`, or +dictionary supplies exact column-name overrides. Unlisted columns use PLAIN, or use +adaptive dictionary encoding when `dictionary=true`. + +## What this package does not support + +Complete coverage of the format is not a goal, so a few parts are deliberately left +out. Reading one of them reports an unsupported feature rather than an invalid file. + +- LZO compression, in either direction. Every available implementation is GPL-2 and + this package is MIT. +- INT96 columns, in either direction. The type is deprecated in the format. +- Writing the deprecated LZ4 codec or the deprecated BIT_PACKED encoding. Both are + still read, because files in the wild use them. New files use `:lz4_raw` and RLE. + +## Statistics + +The writer emits row-group statistics and a complete column-order declaration by +default. Set `statistics=false` to omit both. + +```julia +Parquet.write("without-statistics.parquet", table; statistics=false) +``` + +`Parquet.Limits.max_statistics_value_bytes` limits each raw minimum or maximum +before it is copied into footer metadata. The default is 4096 bytes. If one bound +is larger, the writer omits the minimum and maximum for that column chunk. It still +emits valid null and value counts and preserves column order. + +```julia +limits = Parquet.Limits(max_statistics_value_bytes=1024) +Parquet.write("bounded.parquet", table; limits=limits) +``` + +Readers treat statistics as untrusted metadata. They validate physical widths, +logical ordering, and declared sort order before exposing a bound. + +## Logical values + +Use `Parquet.LogicalColumn` when a Julia element type does not contain the complete +Parquet schema. It can select ENUM, TIME, TIMESTAMP, or DECIMAL metadata. It also +supplies a schema for empty and all-null columns. + +`Parquet.JSONValue` and `Parquet.BSONValue` preserve encoded documents. They validate +their input without building an object tree. `Parquet.Decimal`, `Parquet.Timestamp`, +and `Parquet.Interval` preserve exact Parquet values that have no single matching +Julia standard-library type. + +### Annotations that a read and write cycle does not preserve + +A Julia element type cannot always carry the complete Parquet annotation, so some +columns are written back with a different annotation. Values are still exact. + +- A millisecond `TIMESTAMP` reads as `Parquet.Timestamp{:millis}` when + `isAdjustedToUTC` is true, and as `Dates.DateTime` when it is false. Microsecond + and nanosecond timestamps always read as `Parquet.Timestamp`. +- An `ENUM` column reads as `String` and is written back as `STRING`. +- A millisecond or microsecond `TIME` column reads as `Dates.Time` and is written + back as `TIME` with nanosecond units. +- A `Vector{UInt8}` column is written as `BYTE_ARRAY`. A list of `UInt8` elements is + written as a `LIST` of 8-bit integers. + +Use `Parquet.LogicalColumn` to select the exact annotation when it matters. + +## Resource limits + +Pass a `Parquet.Limits` value to `Parquet.File`, `Parquet.Table`, or `Parquet.write`. +Limits reject oversized metadata, pages, strings, decimals, statistics bounds, and +materialized values before large allocations occur. + +`max_schema_name_bytes` bounds the new top-level column names a single operation may +intern as Julia `Symbol`s. Interned names are process-permanent, so the registry also +keeps a cumulative byte count, but one file's names never consume a later +operation's budget. + +### Nesting depth + +Schema parsing and write validation are iterative and accept very deep schemas under a +raised `max_metadata_depth`. Nested reading still recurses once per level, so it +rejects a plan deeper than 1024 levels with a `LimitError` instead of exhausting the +stack. The writer can therefore produce a synthetic file that the reader declines. +Ordinary Parquet nesting is far below this bound. + +## Sources and file lifetime + +`Parquet.File` and `Parquet.Table` accept a path, a byte vector, or an `IO`. A path is +memory mapped. `close` releases the file descriptor, but the mapping itself lives until +the garbage collector finalizes it. On Windows the file may therefore stay locked after +`close`. If another process truncates a mapped file while it is open, reads of the +removed region terminate the process, and no bounds check can prevent that. + +`max_materialized_bytes` covers the bytes this package allocates. A byte vector is +borrowed and a mapped path is not copied, so neither is charged. An `IO` source is +copied and is charged. A source type supplied by another package owns its own storage, +so bytes it allocates are not charged here. diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..1b7cc93 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,36 @@ +# Parquet.jl + +Parquet.jl is a pure-Julia reader and writer for the Apache Parquet columnar file +format. The package implements the Tables.jl interface and keeps its public API +small and namespaced. + +!!! warning "Development branch" + The `rewrite/1.0` branch is active development work. Do not use it for + production data until its conformance gates are complete. + +## Quick start + +```@example quickstart +using Parquet +using Tables + +path = joinpath(mktempdir(), "example.parquet") +source = (id=Int64[1, 2], label=Union{Missing,String}["alpha", missing]) +Parquet.write(path, source) + +table = Parquet.Table(path) +columns = Tables.columntable(table) +result = (id=collect(columns.id), label=collect(columns.label)) +close(table) +result +``` + +See the [guide](@ref "Guide") for writer options, statistics, and resource limits. +See the [API reference](@ref "API reference") for the public types and functions. + +## Format target + +The stable contract is Apache Parquet format 2.13.0. The reader accepts stable +encodings and codecs, including deprecated input where practical. The writer emits +nondeprecated stable forms. The core does not depend on Arrow.jl or a native Thrift +compiler. diff --git a/src/PAR2/PAR2.jl b/src/PAR2/PAR2.jl deleted file mode 100644 index 338582a..0000000 --- a/src/PAR2/PAR2.jl +++ /dev/null @@ -1,80 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - - -module PAR2 - -using Thrift -import Thrift.process, Thrift.meta, Thrift.distribute - - - - -export meta -export _Type # enum -export ConvertedType # enum -export FieldRepetitionType # enum -export Encoding # enum -export CompressionCodec # enum -export PageType # enum -export BoundaryOrder # enum -export Statistics # struct -export StringType # struct -export UUIDType # struct -export MapType # struct -export ListType # struct -export EnumType # struct -export DateType # struct -export NullType # struct -export DecimalType # struct -export MilliSeconds # struct -export MicroSeconds # struct -export NanoSeconds # struct -export TimeUnit # struct -export TimestampType # struct -export TimeType # struct -export IntType # struct -export JsonType # struct -export BsonType # struct -export LogicalType # struct -export SchemaElement # struct -export DataPageHeader # struct -export IndexPageHeader # struct -export DictionaryPageHeader # struct -export DataPageHeaderV2 # struct -export SplitBlockAlgorithm # struct -export BloomFilterAlgorithm # struct -export XxHash # struct -export BloomFilterHash # struct -export Uncompressed # struct -export BloomFilterCompression # struct -export BloomFilterHeader # struct -export PageHeader # struct -export KeyValue # struct -export SortingColumn # struct -export PageEncodingStats # struct -export ColumnMetaData # struct -export EncryptionWithFooterKey # struct -export EncryptionWithColumnKey # struct -export ColumnCryptoMetaData # struct -export ColumnChunk # struct -export RowGroup # struct -export TypeDefinedOrder # struct -export ColumnOrder # struct -export PageLocation # struct -export OffsetIndex # struct -export ColumnIndex # struct -export AesGcmV1 # struct -export AesGcmCtrV1 # struct -export EncryptionAlgorithm # struct -export FileMetaData # struct -export FileCryptoMetaData # struct - -include("PAR2_constants.jl") -include("PAR2_types.jl") -include("PAR2_impl.jl") # server methods to be hand coded - - -end # module PAR2 diff --git a/src/PAR2/PAR2_constants.jl b/src/PAR2/PAR2_constants.jl deleted file mode 100644 index 26fbdf7..0000000 --- a/src/PAR2/PAR2_constants.jl +++ /dev/null @@ -1,5 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - diff --git a/src/PAR2/PAR2_impl.jl b/src/PAR2/PAR2_impl.jl deleted file mode 100644 index 6dfc7e0..0000000 --- a/src/PAR2/PAR2_impl.jl +++ /dev/null @@ -1,5 +0,0 @@ -# give nicer names to entities that had name clashes -const PARType = _Type -export PARType - -# server implementation intentionally empty as this is just the client part diff --git a/src/PAR2/PAR2_types.jl b/src/PAR2/PAR2_types.jl deleted file mode 100644 index 5bfe1e4..0000000 --- a/src/PAR2/PAR2_types.jl +++ /dev/null @@ -1,2103 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# COV_EXCL_START - -struct _enum__Type - BOOLEAN::Int32 - INT32::Int32 - INT64::Int32 - INT96::Int32 - FLOAT::Int32 - DOUBLE::Int32 - BYTE_ARRAY::Int32 - FIXED_LEN_BYTE_ARRAY::Int32 -end -const _Type = _enum__Type(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4), Int32(5), Int32(6), Int32(7)) - -struct _enum_ConvertedType - UTF8::Int32 - MAP::Int32 - MAP_KEY_VALUE::Int32 - LIST::Int32 - ENUM::Int32 - DECIMAL::Int32 - DATE::Int32 - TIME_MILLIS::Int32 - TIME_MICROS::Int32 - TIMESTAMP_MILLIS::Int32 - TIMESTAMP_MICROS::Int32 - UINT_8::Int32 - UINT_16::Int32 - UINT_32::Int32 - UINT_64::Int32 - INT_8::Int32 - INT_16::Int32 - INT_32::Int32 - INT_64::Int32 - JSON::Int32 - BSON::Int32 - INTERVAL::Int32 -end -const ConvertedType = _enum_ConvertedType(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4), Int32(5), Int32(6), Int32(7), Int32(8), Int32(9), Int32(10), Int32(11), Int32(12), Int32(13), Int32(14), Int32(15), Int32(16), Int32(17), Int32(18), Int32(19), Int32(20), Int32(21)) - -struct _enum_FieldRepetitionType - REQUIRED::Int32 - OPTIONAL::Int32 - REPEATED::Int32 -end -const FieldRepetitionType = _enum_FieldRepetitionType(Int32(0), Int32(1), Int32(2)) - -struct _enum_Encoding - PLAIN::Int32 - PLAIN_DICTIONARY::Int32 - RLE::Int32 - BIT_PACKED::Int32 - DELTA_BINARY_PACKED::Int32 - DELTA_LENGTH_BYTE_ARRAY::Int32 - DELTA_BYTE_ARRAY::Int32 - RLE_DICTIONARY::Int32 - BYTE_STREAM_SPLIT::Int32 -end -const Encoding = _enum_Encoding(Int32(0), Int32(2), Int32(3), Int32(4), Int32(5), Int32(6), Int32(7), Int32(8), Int32(9)) - -struct _enum_CompressionCodec - UNCOMPRESSED::Int32 - SNAPPY::Int32 - GZIP::Int32 - LZO::Int32 - BROTLI::Int32 - LZ4::Int32 - ZSTD::Int32 -end -const CompressionCodec = _enum_CompressionCodec(Int32(0), Int32(1), Int32(2), Int32(3), Int32(4), Int32(5), Int32(6)) - -struct _enum_PageType - DATA_PAGE::Int32 - INDEX_PAGE::Int32 - DICTIONARY_PAGE::Int32 - DATA_PAGE_V2::Int32 -end -const PageType = _enum_PageType(Int32(0), Int32(1), Int32(2), Int32(3)) - -struct _enum_BoundaryOrder - UNORDERED::Int32 - ASCENDING::Int32 - DESCENDING::Int32 -end -const BoundaryOrder = _enum_BoundaryOrder(Int32(0), Int32(1), Int32(2)) - - -mutable struct Statistics <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function Statistics(; kwargs...) - obj = new(__meta__Statistics, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct Statistics - -const __meta__Statistics = meta(Statistics, - Symbol[:max,:min,:null_count,:distinct_count,:max_value,:min_value], - Type[Vector{UInt8},Vector{UInt8},Int64,Int64,Vector{UInt8},Vector{UInt8}], - Symbol[:max,:min,:null_count,:distinct_count,:max_value,:min_value], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::Statistics, name::Symbol) - if name === :max - return (obj.values[name])::Vector{UInt8} - elseif name === :min - return (obj.values[name])::Vector{UInt8} - elseif name === :null_count - return (obj.values[name])::Int64 - elseif name === :distinct_count - return (obj.values[name])::Int64 - elseif name === :max_value - return (obj.values[name])::Vector{UInt8} - elseif name === :min_value - return (obj.values[name])::Vector{UInt8} - else - getfield(obj, name) - end -end - -meta(::Type{Statistics}) = __meta__Statistics - - -mutable struct StringType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function StringType(; kwargs...) - obj = new(__meta__StringType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct StringType - -const __meta__StringType = meta(StringType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{StringType}) = __meta__StringType - - -mutable struct UUIDType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function UUIDType(; kwargs...) - obj = new(__meta__UUIDType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct UUIDType - -const __meta__UUIDType = meta(UUIDType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{UUIDType}) = __meta__UUIDType - - -mutable struct MapType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function MapType(; kwargs...) - obj = new(__meta__MapType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct MapType - -const __meta__MapType = meta(MapType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{MapType}) = __meta__MapType - - -mutable struct ListType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function ListType(; kwargs...) - obj = new(__meta__ListType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct ListType - -const __meta__ListType = meta(ListType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{ListType}) = __meta__ListType - - -mutable struct EnumType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function EnumType(; kwargs...) - obj = new(__meta__EnumType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct EnumType - -const __meta__EnumType = meta(EnumType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{EnumType}) = __meta__EnumType - - -mutable struct DateType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function DateType(; kwargs...) - obj = new(__meta__DateType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct DateType - -const __meta__DateType = meta(DateType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{DateType}) = __meta__DateType - - -mutable struct NullType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function NullType(; kwargs...) - obj = new(__meta__NullType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct NullType - -const __meta__NullType = meta(NullType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{NullType}) = __meta__NullType - - -mutable struct DecimalType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function DecimalType(; kwargs...) - obj = new(__meta__DecimalType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct DecimalType - -const __meta__DecimalType = meta(DecimalType, - Symbol[:scale,:precision], - Type[Int32,Int32], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::DecimalType, name::Symbol) - if name === :scale - return (obj.values[name])::Int32 - elseif name === :precision - return (obj.values[name])::Int32 - else - getfield(obj, name) - end -end - -meta(::Type{DecimalType}) = __meta__DecimalType - - -mutable struct MilliSeconds <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function MilliSeconds(; kwargs...) - obj = new(__meta__MilliSeconds, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct MilliSeconds - -const __meta__MilliSeconds = meta(MilliSeconds, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{MilliSeconds}) = __meta__MilliSeconds - - -mutable struct MicroSeconds <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function MicroSeconds(; kwargs...) - obj = new(__meta__MicroSeconds, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct MicroSeconds - -const __meta__MicroSeconds = meta(MicroSeconds, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{MicroSeconds}) = __meta__MicroSeconds - - -mutable struct NanoSeconds <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function NanoSeconds(; kwargs...) - obj = new(__meta__NanoSeconds, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct NanoSeconds - -const __meta__NanoSeconds = meta(NanoSeconds, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{NanoSeconds}) = __meta__NanoSeconds - - -mutable struct TimeUnit <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function TimeUnit(; kwargs...) - obj = new(__meta__TimeUnit, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct TimeUnit - -const __meta__TimeUnit = meta(TimeUnit, - Symbol[:MILLIS,:MICROS,:NANOS], - Type[MilliSeconds,MicroSeconds,NanoSeconds], - Symbol[:MILLIS,:MICROS,:NANOS], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::TimeUnit, name::Symbol) - if name === :MILLIS - return (obj.values[name])::MilliSeconds - elseif name === :MICROS - return (obj.values[name])::MicroSeconds - elseif name === :NANOS - return (obj.values[name])::NanoSeconds - else - getfield(obj, name) - end -end - -meta(::Type{TimeUnit}) = __meta__TimeUnit - - -mutable struct TimestampType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function TimestampType(; kwargs...) - obj = new(__meta__TimestampType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct TimestampType - -const __meta__TimestampType = meta(TimestampType, - Symbol[:isAdjustedToUTC,:unit], - Type[Bool,TimeUnit], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::TimestampType, name::Symbol) - if name === :isAdjustedToUTC - return (obj.values[name])::Bool - elseif name === :unit - return (obj.values[name])::TimeUnit - else - getfield(obj, name) - end -end - -meta(::Type{TimestampType}) = __meta__TimestampType - - -mutable struct TimeType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function TimeType(; kwargs...) - obj = new(__meta__TimeType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct TimeType - -const __meta__TimeType = meta(TimeType, - Symbol[:isAdjustedToUTC,:unit], - Type[Bool,TimeUnit], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::TimeType, name::Symbol) - if name === :isAdjustedToUTC - return (obj.values[name])::Bool - elseif name === :unit - return (obj.values[name])::TimeUnit - else - getfield(obj, name) - end -end - -meta(::Type{TimeType}) = __meta__TimeType - - -mutable struct IntType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function IntType(; kwargs...) - obj = new(__meta__IntType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct IntType - -const __meta__IntType = meta(IntType, - Symbol[:bitWidth,:isSigned], - Type[UInt8,Bool], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::IntType, name::Symbol) - if name === :bitWidth - return (obj.values[name])::UInt8 - elseif name === :isSigned - return (obj.values[name])::Bool - else - getfield(obj, name) - end -end - -meta(::Type{IntType}) = __meta__IntType - - -mutable struct JsonType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function JsonType(; kwargs...) - obj = new(__meta__JsonType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct JsonType - -const __meta__JsonType = meta(JsonType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{JsonType}) = __meta__JsonType - - -mutable struct BsonType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function BsonType(; kwargs...) - obj = new(__meta__BsonType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct BsonType - -const __meta__BsonType = meta(BsonType, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{BsonType}) = __meta__BsonType - - -mutable struct LogicalType <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function LogicalType(; kwargs...) - obj = new(__meta__LogicalType, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct LogicalType - -const __meta__LogicalType = meta(LogicalType, - Symbol[:STRING,:MAP,:LIST,:ENUM,:DECIMAL,:DATE,:TIME,:TIMESTAMP,:INTEGER,:UNKNOWN,:JSON,:BSON,:UUID], - Type[StringType,MapType,ListType,EnumType,DecimalType,DateType,TimeType,TimestampType,IntType,NullType,JsonType,BsonType,UUIDType], - Symbol[:STRING,:MAP,:LIST,:ENUM,:DECIMAL,:DATE,:TIME,:TIMESTAMP,:INTEGER,:UNKNOWN,:JSON,:BSON,:UUID], - Int[1,2,3,4,5,6,7,8,10,11,12,13,14], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::LogicalType, name::Symbol) - if name === :STRING - return (obj.values[name])::StringType - elseif name === :MAP - return (obj.values[name])::MapType - elseif name === :LIST - return (obj.values[name])::ListType - elseif name === :ENUM - return (obj.values[name])::EnumType - elseif name === :DECIMAL - return (obj.values[name])::DecimalType - elseif name === :DATE - return (obj.values[name])::DateType - elseif name === :TIME - return (obj.values[name])::TimeType - elseif name === :TIMESTAMP - return (obj.values[name])::TimestampType - elseif name === :INTEGER - return (obj.values[name])::IntType - elseif name === :UNKNOWN - return (obj.values[name])::NullType - elseif name === :JSON - return (obj.values[name])::JsonType - elseif name === :BSON - return (obj.values[name])::BsonType - elseif name === :UUID - return (obj.values[name])::UUIDType - else - getfield(obj, name) - end -end - -meta(::Type{LogicalType}) = __meta__LogicalType - - -mutable struct SchemaElement <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function SchemaElement(; kwargs...) - obj = new(__meta__SchemaElement, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct SchemaElement - -const __meta__SchemaElement = meta(SchemaElement, - Symbol[:_type,:type_length,:repetition_type,:name,:num_children,:converted_type,:scale,:precision,:field_id,:logicalType], - Type[Int32,Int32,Int32,String,Int32,Int32,Int32,Int32,Int32,LogicalType], - Symbol[:_type,:type_length,:repetition_type,:num_children,:converted_type,:scale,:precision,:field_id,:logicalType], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::SchemaElement, name::Symbol) - if name === :_type - return (obj.values[name])::Int32 - elseif name === :type_length - return (obj.values[name])::Int32 - elseif name === :repetition_type - return (obj.values[name])::Int32 - elseif name === :name - return (obj.values[name])::String - elseif name === :num_children - return (obj.values[name])::Int32 - elseif name === :converted_type - return (obj.values[name])::Int32 - elseif name === :scale - return (obj.values[name])::Int32 - elseif name === :precision - return (obj.values[name])::Int32 - elseif name === :field_id - return (obj.values[name])::Int32 - elseif name === :logicalType - return (obj.values[name])::LogicalType - else - getfield(obj, name) - end -end - -meta(::Type{SchemaElement}) = __meta__SchemaElement - - -mutable struct DataPageHeader <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function DataPageHeader(; kwargs...) - obj = new(__meta__DataPageHeader, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct DataPageHeader - -const __meta__DataPageHeader = meta(DataPageHeader, - Symbol[:num_values,:encoding,:definition_level_encoding,:repetition_level_encoding,:statistics], - Type[Int32,Int32,Int32,Int32,Statistics], - Symbol[:statistics], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::DataPageHeader, name::Symbol) - if name === :num_values - return (obj.values[name])::Int32 - elseif name === :encoding - return (obj.values[name])::Int32 - elseif name === :definition_level_encoding - return (obj.values[name])::Int32 - elseif name === :repetition_level_encoding - return (obj.values[name])::Int32 - elseif name === :statistics - return (obj.values[name])::Statistics - else - getfield(obj, name) - end -end - -meta(::Type{DataPageHeader}) = __meta__DataPageHeader - - -mutable struct IndexPageHeader <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function IndexPageHeader(; kwargs...) - obj = new(__meta__IndexPageHeader, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct IndexPageHeader - -const __meta__IndexPageHeader = meta(IndexPageHeader, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{IndexPageHeader}) = __meta__IndexPageHeader - - -mutable struct DictionaryPageHeader <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function DictionaryPageHeader(; kwargs...) - obj = new(__meta__DictionaryPageHeader, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct DictionaryPageHeader - -const __meta__DictionaryPageHeader = meta(DictionaryPageHeader, - Symbol[:num_values,:encoding,:is_sorted], - Type[Int32,Int32,Bool], - Symbol[:is_sorted], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::DictionaryPageHeader, name::Symbol) - if name === :num_values - return (obj.values[name])::Int32 - elseif name === :encoding - return (obj.values[name])::Int32 - elseif name === :is_sorted - return (obj.values[name])::Bool - else - getfield(obj, name) - end -end - -meta(::Type{DictionaryPageHeader}) = __meta__DictionaryPageHeader - - -mutable struct DataPageHeaderV2 <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function DataPageHeaderV2(; kwargs...) - obj = new(__meta__DataPageHeaderV2, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct DataPageHeaderV2 - -const __meta__DataPageHeaderV2 = meta(DataPageHeaderV2, - Symbol[:num_values,:num_nulls,:num_rows,:encoding,:definition_levels_byte_length,:repetition_levels_byte_length,:is_compressed,:statistics], - Type[Int32,Int32,Int32,Int32,Int32,Int32,Bool,Statistics], - Symbol[:is_compressed,:statistics], - Int[], - Dict{Symbol,Any}(:is_compressed => true) -) - -function Base.getproperty(obj::DataPageHeaderV2, name::Symbol) - if name === :num_values - return (obj.values[name])::Int32 - elseif name === :num_nulls - return (obj.values[name])::Int32 - elseif name === :num_rows - return (obj.values[name])::Int32 - elseif name === :encoding - return (obj.values[name])::Int32 - elseif name === :definition_levels_byte_length - return (obj.values[name])::Int32 - elseif name === :repetition_levels_byte_length - return (obj.values[name])::Int32 - elseif name === :is_compressed - return (obj.values[name])::Bool - elseif name === :statistics - return (obj.values[name])::Statistics - else - getfield(obj, name) - end -end - -meta(::Type{DataPageHeaderV2}) = __meta__DataPageHeaderV2 - - -mutable struct SplitBlockAlgorithm <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function SplitBlockAlgorithm(; kwargs...) - obj = new(__meta__SplitBlockAlgorithm, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct SplitBlockAlgorithm - -const __meta__SplitBlockAlgorithm = meta(SplitBlockAlgorithm, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{SplitBlockAlgorithm}) = __meta__SplitBlockAlgorithm - - -mutable struct BloomFilterAlgorithm <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function BloomFilterAlgorithm(; kwargs...) - obj = new(__meta__BloomFilterAlgorithm, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct BloomFilterAlgorithm - -const __meta__BloomFilterAlgorithm = meta(BloomFilterAlgorithm, - Symbol[:BLOCK], - Type[SplitBlockAlgorithm], - Symbol[:BLOCK], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::BloomFilterAlgorithm, name::Symbol) - if name === :BLOCK - return (obj.values[name])::SplitBlockAlgorithm - else - getfield(obj, name) - end -end - -meta(::Type{BloomFilterAlgorithm}) = __meta__BloomFilterAlgorithm - - -mutable struct XxHash <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function XxHash(; kwargs...) - obj = new(__meta__XxHash, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct XxHash - -const __meta__XxHash = meta(XxHash, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{XxHash}) = __meta__XxHash - - -mutable struct BloomFilterHash <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function BloomFilterHash(; kwargs...) - obj = new(__meta__BloomFilterHash, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct BloomFilterHash - -const __meta__BloomFilterHash = meta(BloomFilterHash, - Symbol[:XXHASH], - Type[XxHash], - Symbol[:XXHASH], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::BloomFilterHash, name::Symbol) - if name === :XXHASH - return (obj.values[name])::XxHash - else - getfield(obj, name) - end -end - -meta(::Type{BloomFilterHash}) = __meta__BloomFilterHash - - -mutable struct Uncompressed <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function Uncompressed(; kwargs...) - obj = new(__meta__Uncompressed, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct Uncompressed - -const __meta__Uncompressed = meta(Uncompressed, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{Uncompressed}) = __meta__Uncompressed - - -mutable struct BloomFilterCompression <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function BloomFilterCompression(; kwargs...) - obj = new(__meta__BloomFilterCompression, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct BloomFilterCompression - -const __meta__BloomFilterCompression = meta(BloomFilterCompression, - Symbol[:UNCOMPRESSED], - Type[Uncompressed], - Symbol[:UNCOMPRESSED], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::BloomFilterCompression, name::Symbol) - if name === :UNCOMPRESSED - return (obj.values[name])::Uncompressed - else - getfield(obj, name) - end -end - -meta(::Type{BloomFilterCompression}) = __meta__BloomFilterCompression - - -mutable struct BloomFilterHeader <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function BloomFilterHeader(; kwargs...) - obj = new(__meta__BloomFilterHeader, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct BloomFilterHeader - -const __meta__BloomFilterHeader = meta(BloomFilterHeader, - Symbol[:numBytes,:algorithm,:hash,:compression], - Type[Int32,BloomFilterAlgorithm,BloomFilterHash,BloomFilterCompression], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::BloomFilterHeader, name::Symbol) - if name === :numBytes - return (obj.values[name])::Int32 - elseif name === :algorithm - return (obj.values[name])::BloomFilterAlgorithm - elseif name === :hash - return (obj.values[name])::BloomFilterHash - elseif name === :compression - return (obj.values[name])::BloomFilterCompression - else - getfield(obj, name) - end -end - -meta(::Type{BloomFilterHeader}) = __meta__BloomFilterHeader - - -mutable struct PageHeader <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function PageHeader(; kwargs...) - obj = new(__meta__PageHeader, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct PageHeader - -const __meta__PageHeader = meta(PageHeader, - Symbol[:_type,:uncompressed_page_size,:compressed_page_size,:crc,:data_page_header,:index_page_header,:dictionary_page_header,:data_page_header_v2], - Type[Int32,Int32,Int32,Int32,DataPageHeader,IndexPageHeader,DictionaryPageHeader,DataPageHeaderV2], - Symbol[:crc,:data_page_header,:index_page_header,:dictionary_page_header,:data_page_header_v2], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::PageHeader, name::Symbol) - if name === :_type - return (obj.values[name])::Int32 - elseif name === :uncompressed_page_size - return (obj.values[name])::Int32 - elseif name === :compressed_page_size - return (obj.values[name])::Int32 - elseif name === :crc - return (obj.values[name])::Int32 - elseif name === :data_page_header - return (obj.values[name])::DataPageHeader - elseif name === :index_page_header - return (obj.values[name])::IndexPageHeader - elseif name === :dictionary_page_header - return (obj.values[name])::DictionaryPageHeader - elseif name === :data_page_header_v2 - return (obj.values[name])::DataPageHeaderV2 - else - getfield(obj, name) - end -end - -meta(::Type{PageHeader}) = __meta__PageHeader - - -mutable struct KeyValue <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function KeyValue(; kwargs...) - obj = new(__meta__KeyValue, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct KeyValue - -const __meta__KeyValue = meta(KeyValue, - Symbol[:key,:value], - Type[String,String], - Symbol[:value], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::KeyValue, name::Symbol) - if name === :key - return (obj.values[name])::String - elseif name === :value - return (obj.values[name])::String - else - getfield(obj, name) - end -end - -meta(::Type{KeyValue}) = __meta__KeyValue - - -mutable struct SortingColumn <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function SortingColumn(; kwargs...) - obj = new(__meta__SortingColumn, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct SortingColumn - -const __meta__SortingColumn = meta(SortingColumn, - Symbol[:column_idx,:descending,:nulls_first], - Type[Int32,Bool,Bool], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::SortingColumn, name::Symbol) - if name === :column_idx - return (obj.values[name])::Int32 - elseif name === :descending - return (obj.values[name])::Bool - elseif name === :nulls_first - return (obj.values[name])::Bool - else - getfield(obj, name) - end -end - -meta(::Type{SortingColumn}) = __meta__SortingColumn - - -mutable struct PageEncodingStats <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function PageEncodingStats(; kwargs...) - obj = new(__meta__PageEncodingStats, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct PageEncodingStats - -const __meta__PageEncodingStats = meta(PageEncodingStats, - Symbol[:page_type,:encoding,:count], - Type[Int32,Int32,Int32], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::PageEncodingStats, name::Symbol) - if name === :page_type - return (obj.values[name])::Int32 - elseif name === :encoding - return (obj.values[name])::Int32 - elseif name === :count - return (obj.values[name])::Int32 - else - getfield(obj, name) - end -end - -meta(::Type{PageEncodingStats}) = __meta__PageEncodingStats - - -mutable struct ColumnMetaData <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function ColumnMetaData(; kwargs...) - obj = new(__meta__ColumnMetaData, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct ColumnMetaData - -const __meta__ColumnMetaData = meta(ColumnMetaData, - Symbol[:_type,:encodings,:path_in_schema,:codec,:num_values,:total_uncompressed_size,:total_compressed_size,:key_value_metadata,:data_page_offset,:index_page_offset,:dictionary_page_offset,:statistics,:encoding_stats,:bloom_filter_offset], - Type[Int32,Vector{Int32},Vector{String},Int32,Int64,Int64,Int64,Vector{KeyValue},Int64,Int64,Int64,Statistics,Vector{PageEncodingStats},Int64], - Symbol[:key_value_metadata,:index_page_offset,:dictionary_page_offset,:statistics,:encoding_stats,:bloom_filter_offset], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::ColumnMetaData, name::Symbol) - if name === :_type - return (obj.values[name])::Int32 - elseif name === :encodings - return (obj.values[name])::Vector{Int32} - elseif name === :path_in_schema - return (obj.values[name])::Vector{String} - elseif name === :codec - return (obj.values[name])::Int32 - elseif name === :num_values - return (obj.values[name])::Int64 - elseif name === :total_uncompressed_size - return (obj.values[name])::Int64 - elseif name === :total_compressed_size - return (obj.values[name])::Int64 - elseif name === :key_value_metadata - return (obj.values[name])::Vector{KeyValue} - elseif name === :data_page_offset - return (obj.values[name])::Int64 - elseif name === :index_page_offset - return (obj.values[name])::Int64 - elseif name === :dictionary_page_offset - return (obj.values[name])::Int64 - elseif name === :statistics - return (obj.values[name])::Statistics - elseif name === :encoding_stats - return (obj.values[name])::Vector{PageEncodingStats} - elseif name === :bloom_filter_offset - return (obj.values[name])::Int64 - else - getfield(obj, name) - end -end - -meta(::Type{ColumnMetaData}) = __meta__ColumnMetaData - - -mutable struct EncryptionWithFooterKey <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function EncryptionWithFooterKey(; kwargs...) - obj = new(__meta__EncryptionWithFooterKey, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct EncryptionWithFooterKey - -const __meta__EncryptionWithFooterKey = meta(EncryptionWithFooterKey, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{EncryptionWithFooterKey}) = __meta__EncryptionWithFooterKey - - -mutable struct EncryptionWithColumnKey <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function EncryptionWithColumnKey(; kwargs...) - obj = new(__meta__EncryptionWithColumnKey, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct EncryptionWithColumnKey - -const __meta__EncryptionWithColumnKey = meta(EncryptionWithColumnKey, - Symbol[:path_in_schema,:key_metadata], - Type[Vector{String},Vector{UInt8}], - Symbol[:key_metadata], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::EncryptionWithColumnKey, name::Symbol) - if name === :path_in_schema - return (obj.values[name])::Vector{String} - elseif name === :key_metadata - return (obj.values[name])::Vector{UInt8} - else - getfield(obj, name) - end -end - -meta(::Type{EncryptionWithColumnKey}) = __meta__EncryptionWithColumnKey - - -mutable struct ColumnCryptoMetaData <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function ColumnCryptoMetaData(; kwargs...) - obj = new(__meta__ColumnCryptoMetaData, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct ColumnCryptoMetaData - -const __meta__ColumnCryptoMetaData = meta(ColumnCryptoMetaData, - Symbol[:ENCRYPTION_WITH_FOOTER_KEY,:ENCRYPTION_WITH_COLUMN_KEY], - Type[EncryptionWithFooterKey,EncryptionWithColumnKey], - Symbol[:ENCRYPTION_WITH_FOOTER_KEY,:ENCRYPTION_WITH_COLUMN_KEY], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::ColumnCryptoMetaData, name::Symbol) - if name === :ENCRYPTION_WITH_FOOTER_KEY - return (obj.values[name])::EncryptionWithFooterKey - elseif name === :ENCRYPTION_WITH_COLUMN_KEY - return (obj.values[name])::EncryptionWithColumnKey - else - getfield(obj, name) - end -end - -meta(::Type{ColumnCryptoMetaData}) = __meta__ColumnCryptoMetaData - - -mutable struct ColumnChunk <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function ColumnChunk(; kwargs...) - obj = new(__meta__ColumnChunk, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct ColumnChunk - -const __meta__ColumnChunk = meta(ColumnChunk, - Symbol[:file_path,:file_offset,:meta_data,:offset_index_offset,:offset_index_length,:column_index_offset,:column_index_length,:crypto_metadata,:encrypted_column_metadata], - Type[String,Int64,ColumnMetaData,Int64,Int32,Int64,Int32,ColumnCryptoMetaData,Vector{UInt8}], - Symbol[:file_path,:meta_data,:offset_index_offset,:offset_index_length,:column_index_offset,:column_index_length,:crypto_metadata,:encrypted_column_metadata], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::ColumnChunk, name::Symbol) - if name === :file_path - return (obj.values[name])::String - elseif name === :file_offset - return (obj.values[name])::Int64 - elseif name === :meta_data - return (obj.values[name])::ColumnMetaData - elseif name === :offset_index_offset - return (obj.values[name])::Int64 - elseif name === :offset_index_length - return (obj.values[name])::Int32 - elseif name === :column_index_offset - return (obj.values[name])::Int64 - elseif name === :column_index_length - return (obj.values[name])::Int32 - elseif name === :crypto_metadata - return (obj.values[name])::ColumnCryptoMetaData - elseif name === :encrypted_column_metadata - return (obj.values[name])::Vector{UInt8} - else - getfield(obj, name) - end -end - -meta(::Type{ColumnChunk}) = __meta__ColumnChunk - - -mutable struct RowGroup <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function RowGroup(; kwargs...) - obj = new(__meta__RowGroup, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct RowGroup - -const __meta__RowGroup = meta(RowGroup, - Symbol[:columns,:total_byte_size,:num_rows,:sorting_columns,:file_offset,:total_compressed_size,:ordinal], - Type[Vector{ColumnChunk},Int64,Int64,Vector{SortingColumn},Int64,Int64,Int16], - Symbol[:sorting_columns,:file_offset,:total_compressed_size,:ordinal], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::RowGroup, name::Symbol) - if name === :columns - return (obj.values[name])::Vector{ColumnChunk} - elseif name === :total_byte_size - return (obj.values[name])::Int64 - elseif name === :num_rows - return (obj.values[name])::Int64 - elseif name === :sorting_columns - return (obj.values[name])::Vector{SortingColumn} - elseif name === :file_offset - return (obj.values[name])::Int64 - elseif name === :total_compressed_size - return (obj.values[name])::Int64 - elseif name === :ordinal - return (obj.values[name])::Int16 - else - getfield(obj, name) - end -end - -meta(::Type{RowGroup}) = __meta__RowGroup - - -mutable struct TypeDefinedOrder <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function TypeDefinedOrder(; kwargs...) - obj = new(__meta__TypeDefinedOrder, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct TypeDefinedOrder - -const __meta__TypeDefinedOrder = meta(TypeDefinedOrder, - Symbol[], - Type[], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -meta(::Type{TypeDefinedOrder}) = __meta__TypeDefinedOrder - - -mutable struct ColumnOrder <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function ColumnOrder(; kwargs...) - obj = new(__meta__ColumnOrder, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct ColumnOrder - -const __meta__ColumnOrder = meta(ColumnOrder, - Symbol[:TYPE_ORDER], - Type[TypeDefinedOrder], - Symbol[:TYPE_ORDER], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::ColumnOrder, name::Symbol) - if name === :TYPE_ORDER - return (obj.values[name])::TypeDefinedOrder - else - getfield(obj, name) - end -end - -meta(::Type{ColumnOrder}) = __meta__ColumnOrder - - -mutable struct PageLocation <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function PageLocation(; kwargs...) - obj = new(__meta__PageLocation, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct PageLocation - -const __meta__PageLocation = meta(PageLocation, - Symbol[:offset,:compressed_page_size,:first_row_index], - Type[Int64,Int32,Int64], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::PageLocation, name::Symbol) - if name === :offset - return (obj.values[name])::Int64 - elseif name === :compressed_page_size - return (obj.values[name])::Int32 - elseif name === :first_row_index - return (obj.values[name])::Int64 - else - getfield(obj, name) - end -end - -meta(::Type{PageLocation}) = __meta__PageLocation - - -mutable struct OffsetIndex <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function OffsetIndex(; kwargs...) - obj = new(__meta__OffsetIndex, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct OffsetIndex - -const __meta__OffsetIndex = meta(OffsetIndex, - Symbol[:page_locations], - Type[Vector{PageLocation}], - Symbol[], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::OffsetIndex, name::Symbol) - if name === :page_locations - return (obj.values[name])::Vector{PageLocation} - else - getfield(obj, name) - end -end - -meta(::Type{OffsetIndex}) = __meta__OffsetIndex - - -mutable struct ColumnIndex <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function ColumnIndex(; kwargs...) - obj = new(__meta__ColumnIndex, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct ColumnIndex - -const __meta__ColumnIndex = meta(ColumnIndex, - Symbol[:null_pages,:min_values,:max_values,:boundary_order,:null_counts], - Type[Vector{Bool},Vector{Vector{UInt8}},Vector{Vector{UInt8}},Int32,Vector{Int64}], - Symbol[:null_counts], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::ColumnIndex, name::Symbol) - if name === :null_pages - return (obj.values[name])::Vector{Bool} - elseif name === :min_values - return (obj.values[name])::Vector{Vector{UInt8}} - elseif name === :max_values - return (obj.values[name])::Vector{Vector{UInt8}} - elseif name === :boundary_order - return (obj.values[name])::Int32 - elseif name === :null_counts - return (obj.values[name])::Vector{Int64} - else - getfield(obj, name) - end -end - -meta(::Type{ColumnIndex}) = __meta__ColumnIndex - - -mutable struct AesGcmV1 <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function AesGcmV1(; kwargs...) - obj = new(__meta__AesGcmV1, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct AesGcmV1 - -const __meta__AesGcmV1 = meta(AesGcmV1, - Symbol[:aad_prefix,:aad_file_unique,:supply_aad_prefix], - Type[Vector{UInt8},Vector{UInt8},Bool], - Symbol[:aad_prefix,:aad_file_unique,:supply_aad_prefix], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::AesGcmV1, name::Symbol) - if name === :aad_prefix - return (obj.values[name])::Vector{UInt8} - elseif name === :aad_file_unique - return (obj.values[name])::Vector{UInt8} - elseif name === :supply_aad_prefix - return (obj.values[name])::Bool - else - getfield(obj, name) - end -end - -meta(::Type{AesGcmV1}) = __meta__AesGcmV1 - - -mutable struct AesGcmCtrV1 <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function AesGcmCtrV1(; kwargs...) - obj = new(__meta__AesGcmCtrV1, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct AesGcmCtrV1 - -const __meta__AesGcmCtrV1 = meta(AesGcmCtrV1, - Symbol[:aad_prefix,:aad_file_unique,:supply_aad_prefix], - Type[Vector{UInt8},Vector{UInt8},Bool], - Symbol[:aad_prefix,:aad_file_unique,:supply_aad_prefix], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::AesGcmCtrV1, name::Symbol) - if name === :aad_prefix - return (obj.values[name])::Vector{UInt8} - elseif name === :aad_file_unique - return (obj.values[name])::Vector{UInt8} - elseif name === :supply_aad_prefix - return (obj.values[name])::Bool - else - getfield(obj, name) - end -end - -meta(::Type{AesGcmCtrV1}) = __meta__AesGcmCtrV1 - - -mutable struct EncryptionAlgorithm <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function EncryptionAlgorithm(; kwargs...) - obj = new(__meta__EncryptionAlgorithm, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct EncryptionAlgorithm - -const __meta__EncryptionAlgorithm = meta(EncryptionAlgorithm, - Symbol[:AES_GCM_V1,:AES_GCM_CTR_V1], - Type[AesGcmV1,AesGcmCtrV1], - Symbol[:AES_GCM_V1,:AES_GCM_CTR_V1], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::EncryptionAlgorithm, name::Symbol) - if name === :AES_GCM_V1 - return (obj.values[name])::AesGcmV1 - elseif name === :AES_GCM_CTR_V1 - return (obj.values[name])::AesGcmCtrV1 - else - getfield(obj, name) - end -end - -meta(::Type{EncryptionAlgorithm}) = __meta__EncryptionAlgorithm - - -mutable struct FileMetaData <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function FileMetaData(; kwargs...) - obj = new(__meta__FileMetaData, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct FileMetaData - -const __meta__FileMetaData = meta(FileMetaData, - Symbol[:version,:schema,:num_rows,:row_groups,:key_value_metadata,:created_by,:column_orders,:encryption_algorithm,:footer_signing_key_metadata], - Type[Int32,Vector{SchemaElement},Int64,Vector{RowGroup},Vector{KeyValue},String,Vector{ColumnOrder},EncryptionAlgorithm,Vector{UInt8}], - Symbol[:key_value_metadata,:created_by,:column_orders,:encryption_algorithm,:footer_signing_key_metadata], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::FileMetaData, name::Symbol) - if name === :version - return (obj.values[name])::Int32 - elseif name === :schema - return (obj.values[name])::Vector{SchemaElement} - elseif name === :num_rows - return (obj.values[name])::Int64 - elseif name === :row_groups - return (obj.values[name])::Vector{RowGroup} - elseif name === :key_value_metadata - return (obj.values[name])::Vector{KeyValue} - elseif name === :created_by - return (obj.values[name])::String - elseif name === :column_orders - return (obj.values[name])::Vector{ColumnOrder} - elseif name === :encryption_algorithm - return (obj.values[name])::EncryptionAlgorithm - elseif name === :footer_signing_key_metadata - return (obj.values[name])::Vector{UInt8} - else - getfield(obj, name) - end -end - -meta(::Type{FileMetaData}) = __meta__FileMetaData - - -mutable struct FileCryptoMetaData <: Thrift.TMsg - meta::ThriftMeta - values::Dict{Symbol,Any} - - function FileCryptoMetaData(; kwargs...) - obj = new(__meta__FileCryptoMetaData, Dict{Symbol,Any}()) - values = obj.values - symdict = obj.meta.symdict - for nv in kwargs - fldname, fldval = nv - fldtype = symdict[fldname].jtype - (fldname in keys(symdict)) || error(string(typeof(obj), " has no field with name ", fldname)) - values[fldname] = isa(fldval, fldtype) ? fldval : convert(fldtype, fldval) - end - Thrift.setdefaultproperties!(obj) - obj - end -end # mutable struct FileCryptoMetaData - -const __meta__FileCryptoMetaData = meta(FileCryptoMetaData, - Symbol[:encryption_algorithm,:key_metadata], - Type[EncryptionAlgorithm,Vector{UInt8}], - Symbol[:key_metadata], - Int[], - Dict{Symbol,Any}() -) - -function Base.getproperty(obj::FileCryptoMetaData, name::Symbol) - if name === :encryption_algorithm - return (obj.values[name])::EncryptionAlgorithm - elseif name === :key_metadata - return (obj.values[name])::Vector{UInt8} - else - getfield(obj, name) - end -end - -meta(::Type{FileCryptoMetaData}) = __meta__FileCryptoMetaData - -# COV_EXCL_STOP diff --git a/src/Parquet.jl b/src/Parquet.jl index 3cc6a32..b3d8cf3 100644 --- a/src/Parquet.jl +++ b/src/Parquet.jl @@ -1,52 +1,48 @@ module Parquet -using Thrift -using Snappy -using CodecZlib -using CodecZstd -using Dates -using Decimals using Mmap -using Base.Threads -using SentinelArrays +import CRC32 -if VERSION < v"1.3" - using Missings: nonmissingtype -end - -if VERSION < v"1.5" - Base.signed(::Type{UInt16}) = Int16 - Base.signed(::Type{UInt32}) = Int32 - Base.signed(::Type{UInt64}) = Int64 - Base.signed(::Type{UInt128}) = Int128 +include("errors.jl") +include("thrift.jl") +include("metadata/parquet.jl") +include("schema.jl") +include("nested_schema.jl") +include("logical.jl") +include("logical_temporal.jl") +include("logical_json.jl") +include("logical_bson.jl") +include("logical_binary.jl") +include("logical_decimal.jl") +include("statistics.jl") +include("vectors.jl") +include("dremel.jl") +include("source.jl") +include("footer.jl") +include("plain.jl") +include("rle.jl") +include("delta.jl") +include("bss.jl") +include("checksum.jl") +include("page.jl") +include("codecs.jl") +include("dictionary.jl") +include("column.jl") +include("nested_reader.jl") +include("nested_table.jl") +include("write.jl") +include("write_statistics.jl") +include("write_logical.jl") +include("logical_column.jl") +include("write_nested.jl") +include("table.jl") +include("write_provenance.jl") +include("write_splitting.jl") +include("page_index.jl") + +if VERSION >= v"1.11" + Core.eval(@__MODULE__, Expr(:public, :BSONValue, :Decimal, :File, :Interval, + :JSONValue, :Limits, :LogicalColumn, :Table, :Timestamp, :close!, :write)) end -const PARQUET_JL_VERSION = v"0.7.0" - -const _use_mmap = Ref(true) - -function use_mmap(b::Bool) - _use_mmap[] = b end - -import Base: show, open, close, values, eltype, length - -export is_par_file, show, nrows, ncols, rowgroups, columns, pages, bytes, values, colname, colnames -export schema -export logical_timestamp, logical_string -export RecordCursor, BatchedColumnsCursor -export write_parquet, read_parquet - -# package code goes here -include("PAR2/PAR2.jl") -using .PAR2 -include("codec.jl") -include("schema.jl") -include("reader.jl") -include("cursor.jl") -include("writer.jl") -include("simple_reader.jl") -include("dataset.jl") -include("show.jl") - -end # module diff --git a/src/bss.jl b/src/bss.jl new file mode 100644 index 0000000..6de8e41 --- /dev/null +++ b/src/bss.jl @@ -0,0 +1,98 @@ +# BYTE_STREAM_SPLIT (Encodings.md, Parquet 2.13.0): K byte streams of N values each. + +function _splitbits(::Type{Int32}) + return UInt32 +end + +function _splitbits(::Type{Int64}) + return UInt64 +end + +function _splitbits(::Type{Float32}) + return UInt32 +end + +function _splitbits(::Type{Float64}) + return UInt64 +end + +function _splitbytes(count::Integer, width::Integer) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + width >= 0 || throw(ArgumentError("byte width must be nonnegative")) + (count == 0 || width == 0) && return 0 + count <= typemax(Int) ÷ width || throw(FormatError("BYTE_STREAM_SPLIT size overflows")) + return Int(count) * Int(width) +end + +function decode_byte_stream_split!(output::AbstractVector{T}, bytes::AbstractVector{UInt8}; + offset::Integer=1) where {T<:Union{Int32,Int64,Float32,Float64}} + count = length(output) + width = sizeof(T) + total = _splitbytes(count, width) + position = Int(offset) + _requirebytes(bytes, position, total) + U = _splitbits(T) + base = firstindex(output) + @inbounds for index in 0:(count - 1) + value = zero(U) + for stream in 0:(width - 1) + value |= U(bytes[position + stream * count + index]) << (8 * stream) + end + output[base + index] = reinterpret(T, value) + end + return position + total +end + +function decode_byte_stream_split(::Type{T}, bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits()) where {T<:Union{Int32,Int64,Float32,Float64}} + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + _checkbytes(count, sizeof(T), limits) + output = Vector{T}(undef, _structural(count)) + position = decode_byte_stream_split!(output, bytes; offset=offset) + return output, position +end + +function decode_byte_stream_split_fixed(bytes::AbstractVector{UInt8}, count::Integer, width::Integer; + offset::Integer=1, limits::Limits=Limits()) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + width > 0 || throw(FormatError("fixed byte-array width must be positive")) + _checklimit(:container_elements, count, limits.max_container_elements) + _checklimit(:string_bytes, width, limits.max_string_bytes) + total = _splitbytes(count, width) + _checklimit(:page_bytes, total, limits.max_page_bytes) + position = Int(offset) + _requirebytes(bytes, position, total) + size = _structural(count) + fixedwidth = _structural(width) + output = Matrix{UInt8}(undef, fixedwidth, size) + @inbounds for index in 0:(size - 1), stream in 0:(fixedwidth - 1) + output[stream + 1, index + 1] = bytes[position + stream * size + index] + end + return output, position + total +end + +function encode_byte_stream_split(values::AbstractVector{T}) where {T<:Union{Int32,Int64,Float32,Float64}} + count = length(values) + width = sizeof(T) + output = Vector{UInt8}(undef, _splitbytes(count, width)) + U = _splitbits(T) + base = firstindex(values) + @inbounds for index in 0:(count - 1) + bits = reinterpret(U, values[base + index]) + for stream in 0:(width - 1) + output[stream * count + index + 1] = UInt8((bits >> (8 * stream)) & 0xff) + end + end + return output +end + +function encode_byte_stream_split_fixed(values::AbstractMatrix{UInt8}) + width, count = size(values) + width > 0 || throw(ArgumentError("fixed byte-array width must be positive")) + output = Vector{UInt8}(undef, _splitbytes(count, width)) + @inbounds for index in 0:(count - 1), stream in 0:(width - 1) + output[stream * count + index + 1] = values[stream + 1, index + 1] + end + return output +end diff --git a/src/checksum.jl b/src/checksum.jl new file mode 100644 index 0000000..5b1ca63 --- /dev/null +++ b/src/checksum.jl @@ -0,0 +1,26 @@ +function pagechecksum(bytes::AbstractVector{UInt8}) + return CRC32.crc32(bytes) +end + +function _pagechecksumscratch(::CRC32.ByteArray) + return Int64(0) +end + +function _pagechecksumscratch(bytes::AbstractVector{UInt8}) + count = min(length(bytes), 24_576) + return _materializedarraybytes(UInt8, count) +end + +function verifypagechecksum(expected::Int32, + bytes::AbstractVector{UInt8}; + budget::Union{Nothing,_LiveByteBudget}=nothing) + scratch = budget === nothing ? Int64(0) : _pagechecksumscratch(bytes) + iszero(scratch) || _reserve!(budget, scratch) + try + actual = pagechecksum(bytes) + actual == reinterpret(UInt32, expected) && return + throw(FormatError("page CRC32 mismatch")) + finally + iszero(scratch) || _release!(something(budget), scratch) + end +end diff --git a/src/codec.jl b/src/codec.jl deleted file mode 100644 index 27f463d..0000000 --- a/src/codec.jl +++ /dev/null @@ -1,484 +0,0 @@ -# ref: https://github.com/apache/parquet-format/blob/master/Encodings.md - -macro bitwidth(i) - quote - ceil(Int, log(2, $(esc(i))+1)) - end -end -macro bit2bytewidth(i) - quote - ceil(Int, $(esc(i))/8) - end -end -macro byt2itype(i) - quote - ($(esc(i)) <= 4) ? Int32 : ($(esc(i)) <= 8) ? Int64 : Int128 - end -end -macro byt2uitype(i) - quote - ($(esc(i)) <= 4) ? UInt32 : ($(esc(i)) <= 8) ? UInt64 : UInt128 - end -end -macro byt2uitype_small(i) - quote - ($(esc(i)) <= 1) ? UInt8 : ($(esc(i)) <= 2) ? UInt16 : ($(esc(i)) <= 4) ? UInt32 : ($(esc(i)) <= 8) ? UInt64 : UInt128 - end -end - -const MSB = 0x80 -const MASK7 = 0x7f -const MASK8 = 0xff -const MASK3 = 0x07 -function MASKN(nbits::UInt8) - byte_width = @bit2bytewidth(nbits) - type_small = @byt2uitype_small(byte_width) - MASKN(nbits, type_small) -end -function MASKN(nbits::UInt8, ::Type{T}) where {T} - O = convert(T, 0x1) - (O << nbits) - O -end - -#read_fixed(io::IO, typ::Type{UInt32}) = _read_fixed(io, convert(UInt32,0), 4) -#read_fixed(io::IO, typ::Type{UInt64}) = _read_fixed(io, convert(UInt64,0), 8) -read_fixed(io::IO, typ::Type{Int32}) = reinterpret(Int32, _read_fixed(io, convert(UInt32,0), 4)) -#read_fixed(io::IO, typ::Type{Int64}) = reinterpret(Int64, _read_fixed(io, convert(UInt64,0), 8)) -#read_fixed(io::IO, typ::Type{Int128}) = reinterpret(Int128, _read_fixed(io, convert(UInt128, 0), 12)) # INT96: 12 bytes little endian -#read_fixed(io::IO, typ::Type{Float32}) = reinterpret(Float32, _read_fixed(io, convert(UInt32,0), 4)) -#read_fixed(io::IO, typ::Type{Float64}) = reinterpret(Float64, _read_fixed(io, convert(UInt64,0), 8)) -function _read_fixed(io::IO, ret::T, N::Int) where {T <: Unsigned} - for n in 0:(N-1) - byte = convert(T, read(io, UInt8)) - ret |= (byte << *(8,n)) - end - ret -end - -function _read_fixed_bigendian(io::IO, ret::T, N::Int) where {T <: Unsigned} - for n in (N-1):0 - byte = convert(T, read(io, UInt8)) - ret |= (byte << *(8,n)) - end -end - -mutable struct InputState - data::Vector{UInt8} - offset::Int -end - -mutable struct OutputState{T} - data::Vector{T} - offset::Int -end - -function OutputState(::Type{T}, size) where {T} - arr = Array{T}(undef, size) - OutputState{T}(arr, 0) -end - -function ensure_additional_size(iostate, additional_size) - needed_size = iostate.offset + additional_size - if length(iostate.data) < needed_size - resize!(iostate.data, needed_size) - end - nothing -end - -function reset_to_size(iostate, size) - iostate.offset = 0 - (length(iostate.data) == size) || resize!(iostate.data, size) - nothing -end - -read_fixed(inp::InputState, typ::Type{UInt32}) = _read_fixed(inp, convert(UInt32,0), 4) -read_fixed(inp::InputState, typ::Type{UInt64}) = _read_fixed(inp, convert(UInt64,0), 8) -read_fixed(inp::InputState, typ::Type{Int32}) = reinterpret(Int32, _read_fixed(inp, convert(UInt32,0), 4)) -read_fixed(inp::InputState, typ::Type{Int64}) = reinterpret(Int64, _read_fixed(inp, convert(UInt64,0), 8)) -read_fixed(inp::InputState, typ::Type{Int128}) = reinterpret(Int128, _read_fixed(inp, convert(UInt128, 0), 12)) -read_fixed(inp::InputState, typ::Type{Float32}) = reinterpret(Float32, _read_fixed(inp, convert(UInt32,0), 4)) -read_fixed(inp::InputState, typ::Type{Float64}) = reinterpret(Float64, _read_fixed(inp, convert(UInt64,0), 8)) -function _read_fixed(inp::InputState, ret::T, N::Int) where {T <: Unsigned} - data = inp.data - offset = inp.offset - for n in 0:(N-1) - byte = convert(T, data[1+offset]) - offset += 1 - ret |= (byte << *(8,n)) - end - inp.offset = offset - ret -end - -function _read_varint(inp::InputState, ::Type{T}) where {T <: Integer} - data = inp.data - offset = inp.offset - res = zero(T) - n = 0 - byte = UInt8(MSB) - while (byte & MSB) != 0 - byte = data[1+offset] - offset += 1 - res |= (convert(T, byte & MASK7) << (7*n)) - n += 1 - end - inp.offset = offset - - # in case of overflow, consider it as missing field and return default value - if (n-1) > sizeof(T) - #@debug("overflow reading $T. returning 0") - return zero(T) - end - res -end - -# parquet types: BOOLEAN, INT32, INT64, INT96, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY -# enum values: 0, 1, 2, 3, 4, 5, 6, 7 -const PLAIN_JTYPES = (Bool, Int32, Int64, Int128, Float32, Float64, UInt8, UInt8) - -# read plain encoding (PLAIN = 0) -read_plain_byte_array(inp::InputState) = read_plain_byte_array(inp::InputState, read_fixed(inp, Int32)) -function read_plain_byte_array(inp::InputState, count::Int32) - arr = inp.data[(1+inp.offset):(inp.offset+count)] # TODO: Return subarr? - inp.offset += count - arr -end - -# read plain values _Type.BOOLEAN -function read_plain_values(inp::InputState, out::OutputState{Bool}, count::Int32) - #@debug("reading plain values", type=Bool, count=count) - read_bitpacked_booleans(inp, out, count) -end -# read plain values _Type.BYTE_ARRAY -function read_plain_values(inp::InputState, out::OutputState{Vector{UInt8}}, count::Int32) - # _Type.FIXED_LEN_BYTE_ARRAY is most likely same as byte array - #@debug("reading plain values", type=Vector{UInt8}, count=count) - arr = out.data - offset = out.offset - @assert (offset + count) <= length(arr) - @inbounds for i in 1:count - arr[i+offset] = read_plain_byte_array(inp) - end - out.offset += count - nothing -end -function read_plain_values(inp::InputState, out::OutputState{T}, count::Int32, converter_fn::Function, storage_type::Int32) where T <: Union{Decimal,Float64,Int16,Int32,Int64,Int128} - arr = out.data - offset = out.offset - @assert (offset + count) <= length(arr) - if storage_type === _Type.FIXED_LEN_BYTE_ARRAY - elem_bytes_len = Int32((length(inp.data) - inp.offset) / length(arr)) - #@debug("reading decimal plain values", offset, count, length(arr), length(inp.data), inp.offset, elem_bytes_len) - @inbounds for i in 1:count - arr[i+offset] = converter_fn(read_plain_byte_array(inp, elem_bytes_len)) - end - elseif storage_type === _Type.INT64 - @inbounds for i in 1:count - arr[i+offset] = converter_fn(read_fixed(inp, Int64)) - end - elseif storage_type === _Type.INT32 - @inbounds for i in 1:count - arr[i+offset] = converter_fn(read_fixed(inp, Int32)) - end - else - error("unsupported storage type $(storage_type) for $T") - end - out.offset += count - nothing -end -function read_plain_values(inp::InputState, out::OutputState{String}, count::Int32, converter_fn::Function, storage_type::Int32) - #@debug("reading plain values", type=Vector{UInt8}, count=count) - arr = out.data - offset = out.offset - @assert (offset + count) <= length(arr) - if storage_type === _Type.BYTE_ARRAY - @inbounds for i in 1:count - arr[i+offset] = converter_fn(read_plain_byte_array(inp)) - end - elseif storage_type === _Type.FIXED_LEN_BYTE_ARRAY - elem_bytes_len = Int32((length(inp.data) - inp.offset) / length(arr)) - @inbounds for i in 1:count - arr[i+offset] = converter_fn(read_plain_byte_array(inp, elem_bytes_len)) - end - else - error("unsupported storage type $(storage_type) for String") - end - out.offset += count - nothing -end -# read_plain_values of type T -function read_plain_values(inp::InputState, out::OutputState{T}, count::Int32) where {T} - #@debug("reading plain values", type=T, count=count) - arr = out.data - offset = out.offset - @assert (offset + count) <= length(arr) - @inbounds for i in 1:count - arr[i+offset] = read_fixed(inp, T) - end - #@debug("read $(length(arr)) plain values") - out.offset += count - nothing -end -function read_plain_values(inp::InputState, out::OutputState{DateTime}, count::Int32, converter_fn::Function, storage_type::Int32) - #@debug("reading plain values", type=T, count=count) - arr = out.data - offset = out.offset - @assert (offset + count) <= length(arr) - if storage_type === _Type.INT96 - @inbounds for i in 1:count - arr[i+offset] = converter_fn(read_fixed(inp, Int128)) - end - else - error("unsupported storage type $(storage_type) for DateTime") - end - #@debug("read $(length(arr)) plain values") - out.offset += count - nothing -end - -function read_bitpacked_booleans(inp::InputState, out::OutputState{Bool}, count::Int32) - #@debug("reading bitpacked booleans", count) - arrpos = 1 - bits = UInt8(0) - bitpos = 9 - - data = inp.data - data_offset = inp.offset - - arr = out.data - offset = out.offset - @assert (offset+count) <= length(arr) - - @inbounds while arrpos <= count - if bitpos > 8 - bits = data[1+data_offset] - data_offset += 1 - #@debug("bits", bits, bitstring(bits)) - bitpos = 1 - end - arr[arrpos+offset] = Bool(bits & 0x1) - arrpos += 1 - bits >>= 1 - bitpos += 1 - end - out.offset += count - inp.offset = data_offset - nothing -end - -# read data dictionary (RLE_DICTIONARY = 8, or PLAIN_DICTIONARY = 2 in a data page) -function read_data_dict(inp::InputState, count::Int32) - bits = inp.data[inp.offset+=1] - #@info("bits", bits) - byte_width = @bit2bytewidth(bits) - typ = @byt2itype(byte_width) - out = OutputState(typ, count) - - #@info("reading read_hybrid", count, read_len) - read_hybrid(inp, out, count, bits, byte_width; read_len=false) - out.data -end - -# read RLE or bit backed format (RLE = 3) -function read_hybrid(inp::InputState, out::OutputState{T}, count::Int32, bits::UInt8, byt::Int; read_len::Bool=true) where {T} - len = Int32(0) - if read_len - len = read_fixed(inp, Int32) - end - #@debug("reading hybrid data", len, count, bits) - mask = MASKN(bits) - arr = out.data - arrpos = 1 - offset = out.offset - 1 # to counter arrpos starting at 1 - while arrpos <= count - runhdr = _read_varint(inp, Int) - isbitpack = ((runhdr & 0x1) == 0x1) - runhdr >>= 1 - nrunhdrbits = runhdr * 8 - nitems = min(isbitpack ? nrunhdrbits : runhdr, count - arrpos + 1) - - if isbitpack - runcount = min(nrunhdrbits, length(arr)-offset-arrpos) - read_bitpacked_run(inp, out, runcount, bits, byt, mask) - if nrunhdrbits > runcount - nbytes_to_skip = div(nrunhdrbits - runcount, 8) - @debug("skipping trailing bytes in bitpacked run", nbytes_to_skip) - inp.offset += nbytes_to_skip - end - #out.offset += runcount - else # rle - read_type = @byt2uitype(byt) - read_rle_run(inp, out, nitems, bits, byt, read_type) - #out.offset += nitems - end - arrpos += nitems - end - nothing -end - -function read_rle_run(inp::InputState, out::OutputState{T}, count::Int, bits::UInt8, byt::Int, read_type::Type{V}) where {T,V} - #@debug("read_rle_run", count, T, bits, byt) - rawval = _read_fixed(inp, zero(V), byt) - val = reinterpret(T, rawval) - arr = out.data - offset = out.offset - @assert length(arr) >= (count+offset) - @inbounds for idx in 1:count - arr[idx+offset] = val - end - out.offset += count - nothing -end - -function read_bitpacked_run(inp::InputState, out::OutputState{T}, count::Int, bits::UInt8, byt::Int, mask::V=MASKN(bits)) where {T,V} - bitbuff = zero(V) - nbitsbuff = UInt8(0) - shift = UInt8(0) - - data = inp.data - arr = out.data - offset = out.offset - arridx = 1 - dataidx = 1 + inp.offset - while arridx <= count - #@debug("arridx:$arridx nbitsbuff:$nbitsbuff shift:$shift bits:$bits") - if nbitsbuff > 0 - # we have leftover bits, which must be appended - if nbitsbuff < bits - # but only append if we need to read more in this cycle - @inbounds arr[arridx+offset] = bitbuff & MASKN(nbitsbuff, V) - shift = nbitsbuff - nbitsbuff = UInt8(0) - bitbuff = zero(V) - end - end - - # fill buffer - while (nbitsbuff + shift) < bits - # shift 8 bits and read directly into bitbuff - bitbuff |= (V(data[dataidx]) << nbitsbuff) - dataidx += 1 - nbitsbuff += UInt8(8) - end - - # set values - while ((nbitsbuff + shift) >= bits) && (arridx <= count) - if shift > 0 - remshift = bits - shift - #@debug("setting part from bitbuff nbitsbuff:$nbitsbuff, shift:$shift, remshift:$remshift") - arr[arridx+offset] |= convert(T, (bitbuff << shift) & mask) - bitbuff >>= remshift - nbitsbuff -= remshift - shift = UInt8(0) - else - #@debug("setting all from bitbuff nbitsbuff:$nbitsbuff") - arr[arridx+offset] = convert(T, bitbuff & mask) - bitbuff >>= bits - nbitsbuff -= bits - end - arridx += 1 - end - end - inp.offset = dataidx - 1 - out.offset += count - nothing -end - -# read bit packed in deprecated format (BIT_PACKED = 4) -function read_bitpacked_run_old(inp::InputState, out::OutputState{T}, count::Int32, bits::UInt8, mask::V=MASKN(bits)) where {T <: Integer, V <: Integer} - # the mask is of the smallest bounding type for bits - # T is one of the types that map on to the appropriate Julia type in Parquet (which may be larger than the mask type) - bitbuff = zero(V) - nbitsbuff = 0 - - data = inp.data - arr = out.data - offset = out.offset - arridx = Int32(1) - dataidx = Int32(1 + inp.offset) - while arridx <= count - diffnbits = bits - nbitsbuff - while diffnbits > 8 - # shift 8 bits and read directly into bitbuff - bitbuff <<= 8 - bitbuff |= data[dataidx] - dataidx += Int32(1) - nbitsbuff += 8 - diffnbits -= 8 - end - - if diffnbits > 0 - # read next byte from input - nxtdata = data[dataidx] - dataidx += Int32(1) - # shift bitbuff by diffnbits, add diffnbits and set result - nbitsbuff = 8 - diffnbits - arr[arridx+offset] = convert(T, ((bitbuff << diffnbits) | (nxtdata >> nbitsbuff)) & mask) - arridx += Int32(1) - # keep remaining bits in bitbuff - bitbuff <<= 8 - bitbuff |= nxtdata - else - # set result - arr[arridx+offset] = convert(T, (bitbuff >> abs(diffnbits)) & mask) - arridx += Int32(1) - nbitsbuff -= bits - end - end - inp.offset = dataidx - 1 - out.offset += count - nothing -end - -function logical_timestamp(barr; offset::Dates.Period=Dates.Second(0)) - nanos = read(IOBuffer(barr[1:8]), Int64) - julian_days = read(IOBuffer(barr[9:12]), Int32) - Dates.julian2datetime(julian_days) + Dates.Nanosecond(nanos) + offset -end - -function logical_timestamp(i128::Int128; offset::Dates.Period=Dates.Second(0)) - iob = IOBuffer() - write(iob, i128) - logical_timestamp(take!(iob); offset=offset) -end - -logical_string(bytes::Vector{UInt8}) = String(bytes) - -function logical_decimal(bytes::Union{Int64,Int32,Vector{UInt8}}, precision::Integer, scale::Integer; use_float::Bool=false) - T = logical_decimal_unscaled_type(Int32(precision)) - if scale == 0 - logical_decimal_integer(bytes, T) - elseif use_float - logical_decimal_float64(bytes, T, Int32(scale)) - else - logical_decimal_scaled(bytes, T, Int32(scale)) - end -end - -function logical_decimal_integer(intval::Union{Int64,Int32}, ::Type{T}) where T <: Union{UInt16,UInt32,UInt64,UInt128} - signed(T)(intval) -end - -function logical_decimal_integer(bytes::Vector{UInt8}, ::Type{T}) where T <: Union{UInt16,UInt32,UInt64,UInt128} - N = length(bytes) - uintval = T(0) - for idx in 1:N - uintval |= (T(bytes[idx]) << *(8,N-idx)) - end - reinterpret(signed(T), uintval) -end - -function logical_decimal_float64(bytes::Union{Int64,Int32,Vector{UInt8}}, ::Type{T}, scale::Int32) where T <: Union{UInt16,UInt32,UInt64,UInt128} - if scale > 0 - logical_decimal_integer(bytes, T) / 10^scale - else - logical_decimal_integer(bytes, T) * 10^abs(scale) - end -end - -function logical_decimal_scaled(bytes::Union{Int64,Int32,Vector{UInt8}}, ::Type{T}, scale::Int32) where T <: Union{UInt16,UInt32,UInt64,UInt128} - intval = logical_decimal_integer(bytes, T) - sign = (intval < 0) ? 1 : 0 - intval = abs(intval) - Decimal(sign, intval, -scale) -end \ No newline at end of file diff --git a/src/codecs.jl b/src/codecs.jl new file mode 100644 index 0000000..ed68ba8 --- /dev/null +++ b/src/codecs.jl @@ -0,0 +1,288 @@ +# Page compression codecs (Compression.md, Parquet 2.13.0). Every decompression targets the +# exact uncompressed size declared by the page header and never allocates beyond the limits. + +import ChunkCodecCore +import ChunkCodecLibSnappy +import ChunkCodecLibZlib +import ChunkCodecLibZstd +import ChunkCodecLibLz4 +import ChunkCodecLibBrotli + +const SNAPPY_CODEC = ChunkCodecLibSnappy.SnappyCodec() +const GZIP_CODEC = ChunkCodecLibZlib.GzipCodec() +const ZSTD_CODEC = ChunkCodecLibZstd.ZstdCodec() +const LZ4_BLOCK_CODEC = ChunkCodecLibLz4.LZ4BlockCodec() +const BROTLI_CODEC = ChunkCodecLibBrotli.BrotliCodec() +const LZ4_BLOCK_DECODER = ChunkCodecLibLz4.LZ4BlockDecodeOptions() + +function codecname(codec::Metadata.CompressionCodec.T) + name = Thrift.name(codec) + name === nothing && return "CompressionCodec.T($(codec.value))" + return String(name) +end + +function codecreadable(codec::Metadata.CompressionCodec.T) + return codec == Metadata.CompressionCodec.UNCOMPRESSED || codec == Metadata.CompressionCodec.SNAPPY || + codec == Metadata.CompressionCodec.GZIP || codec == Metadata.CompressionCodec.BROTLI || + codec == Metadata.CompressionCodec.LZ4 || codec == Metadata.CompressionCodec.ZSTD || + codec == Metadata.CompressionCodec.LZ4_RAW +end + +# The deprecated LZ4 codec is read-only: new files use LZ4_RAW (Compression.md). +function codecwritable(codec::Metadata.CompressionCodec.T) + return codecreadable(codec) && codec != Metadata.CompressionCodec.LZ4 +end + +# A file that uses LZO is well formed; this package declines to support it, because +# every available LZO implementation is GPL-2 and this core is MIT. That is a +# permanent exclusion rather than pending work. +function _unreadablecodec(codec::Metadata.CompressionCodec.T) + codec == Metadata.CompressionCodec.LZO && throw(UnsupportedFeatureError( + "LZO compression is not supported; no license-compatible implementation exists")) + throw(FormatError("unknown compression codec $(codecname(codec))")) +end + +function _unwritablecodec(codec::Metadata.CompressionCodec.T) + codec == Metadata.CompressionCodec.LZ4 && + throw(ArgumentError("the deprecated LZ4 codec is read-only; write LZ4_RAW instead")) + codec == Metadata.CompressionCodec.LZO && throw(ArgumentError("LZO compression is not supported")) + throw(ArgumentError("unknown compression codec $(codecname(codec))")) +end + +function _contiguous(bytes::Vector{UInt8}) + return bytes +end + +function _contiguous(bytes::SubArray{UInt8,1,Vector{UInt8},Tuple{UnitRange{Int}},true}) + return bytes +end + +function _contiguous(bytes::BufferSlice) + (@atomic bytes.region.closed) && throw(ArgumentError("Parquet byte region is closed")) + first = firstindex(bytes.region.bytes) + Int(bytes.offset) + return _contiguous(view(bytes.region.bytes, first:(first + Int(bytes.count) - 1))) +end + +function _contiguous(bytes::AbstractVector{UInt8}) + return Vector{UInt8}(bytes) +end + +function _contiguouscopycharge(::Vector{UInt8}) + return Int64(0) +end + +function _contiguouscopycharge( + ::SubArray{UInt8,1,Vector{UInt8},Tuple{UnitRange{Int}},true}) + return Int64(0) +end + +function _contiguouscopycharge(bytes::BufferSlice) + (@atomic bytes.region.closed) && throw(ArgumentError("Parquet byte region is closed")) + first = firstindex(bytes.region.bytes) + Int(bytes.offset) + viewbytes = view(bytes.region.bytes, + first:(first + Int(bytes.count) - 1)) + return _contiguouscopycharge(viewbytes) +end + +function _contiguouscopycharge(bytes::AbstractVector{UInt8}) + return _materializedarraybytes(UInt8, length(bytes)) +end + +function _codecfailure(err, codec::Metadata.CompressionCodec.T) + err isa ChunkCodecCore.DecodedSizeError && + throw(FormatError("$(codecname(codec)) page did not decompress to the declared size")) + err isa ChunkCodecCore.DecodingError && + throw(FormatError("$(codecname(codec)) page is corrupt: $(sprint(showerror, err))")) + throw(err) +end + +function _readbe32(bytes::AbstractVector{UInt8}, offset::Int) + value = UInt32(0) + @inbounds for index in 0:3 + value = (value << 8) | UInt32(bytes[offset + index]) + end + return Int64(value) +end + +function _lz4blocksize!(target::AbstractVector{UInt8}, block::AbstractVector{UInt8}) + size = try + ChunkCodecCore.try_decode!(LZ4_BLOCK_DECODER, target, block) + catch err + err isa ChunkCodecCore.DecodingError || rethrow() + return nothing + end + ChunkCodecCore.is_size(size) || return nothing + decoded = Int(size) + 0 < decoded <= length(target) || return nothing + return decoded +end + +function _lz4emptyblock(block::AbstractVector{UInt8}) + size = try + ChunkCodecCore.try_decode!(LZ4_BLOCK_DECODER, UInt8[], block) + catch err + err isa ChunkCodecCore.DecodingError || rethrow() + return false + end + return ChunkCodecCore.is_size(size) && Int(size) == 0 +end + +# Hadoop BlockCompressorStream framing: each block starts with its total uncompressed size, +# followed by one or more compressed-size-prefixed LZ4 chunks. A zero block is an empty stream. +function _lz4hadoop!(output::Vector{UInt8}, source::AbstractVector{UInt8}) + isempty(source) && return false + position = 1 + produced = 1 + while position <= length(source) + length(source) - position + 1 >= 4 || return false + original = _readbe32(source, position) + position += 4 + if original == 0 + return position == length(source) + 1 && produced == length(output) + 1 + end + original <= length(output) - produced + 1 || return false + blockproduced = 0 + while blockproduced < original + length(source) - position + 1 >= 4 || return false + compressed = _readbe32(source, position) + position += 4 + 0 < compressed <= length(source) - position + 1 || return false + block = view(source, position:(position + compressed - 1)) + target = view(output, (produced + blockproduced):(produced + original - 1)) + decoded = _lz4blocksize!(target, block) + decoded === nothing && return false + blockproduced += decoded + position += compressed + end + produced += original + end + return produced == length(output) + 1 +end + +function _lz4arrowhadoop!(output::Vector{UInt8}, source::AbstractVector{UInt8}) + isempty(source) && return false + position = 1 + produced = 1 + while position <= length(source) + length(source) - position + 1 >= 8 || return false + original = _readbe32(source, position) + compressed = _readbe32(source, position + 4) + position += 8 + compressed <= length(source) - position + 1 || return false + original <= length(output) - produced + 1 || return false + if original == 0 + if compressed > 0 + block = view(source, position:(position + compressed - 1)) + _lz4emptyblock(block) || return false + end + else + compressed > 0 || return false + block = view(source, position:(position + compressed - 1)) + target = view(output, produced:(produced + original - 1)) + decoded = _lz4blocksize!(target, block) + decoded == original || return false + end + position += compressed + produced += original + end + return produced == length(output) + 1 +end + +function _decompresslz4!(output::Vector{UInt8}, source::AbstractVector{UInt8}) + _lz4hadoop!(output, source) && return + _lz4arrowhadoop!(output, source) && return + ChunkCodecCore.decode!(LZ4_BLOCK_CODEC, output, source) + return +end + +function _decompress!(codec::Metadata.CompressionCodec.T, output::Vector{UInt8}, source::AbstractVector{UInt8}) + if codec == Metadata.CompressionCodec.SNAPPY + ChunkCodecCore.decode!(SNAPPY_CODEC, output, source) + elseif codec == Metadata.CompressionCodec.GZIP + ChunkCodecCore.decode!(GZIP_CODEC, output, source) + elseif codec == Metadata.CompressionCodec.ZSTD + ChunkCodecCore.decode!(ZSTD_CODEC, output, source) + elseif codec == Metadata.CompressionCodec.BROTLI + ChunkCodecCore.decode!(BROTLI_CODEC, output, source) + elseif codec == Metadata.CompressionCodec.LZ4_RAW + ChunkCodecCore.decode!(LZ4_BLOCK_CODEC, output, source) + else + _decompresslz4!(output, source) + end + return +end + +function _uncompressed(bytes::AbstractVector{UInt8}, expected::Int) + length(bytes) == expected || + throw(FormatError("uncompressed page holds $(length(bytes)) bytes but declares $expected")) + return bytes +end + +""" + decompress(codec, bytes, expected; limits) -> AbstractVector{UInt8} + +Decompress one page payload to exactly `expected` bytes. Sizes are charged to +`limits.max_page_bytes` before allocation, UNCOMPRESSED pages are returned as-is, and any +codec failure or size disagreement is a `FormatError`. +""" +function decompress(codec::Metadata.CompressionCodec.T, + bytes::AbstractVector{UInt8}, expected::Integer; + limits::Limits=Limits(), + budget::Union{Nothing,_LiveByteBudget}=nothing) + expected >= 0 || throw(FormatError("negative uncompressed page size $expected")) + _checklimit(:page_bytes, expected, limits.max_page_bytes) + _checklimit(:page_bytes, length(bytes), limits.max_page_bytes) + expected <= typemax(Int) || throw(FormatError("uncompressed page size $expected overflows")) + codecreadable(codec) || _unreadablecodec(codec) + codec == Metadata.CompressionCodec.UNCOMPRESSED && return _uncompressed(bytes, Int(expected)) + copycharge = budget === nothing ? Int64(0) : _contiguouscopycharge(bytes) + iszero(copycharge) || _reserve!(budget, copycharge) + try + source = _contiguous(bytes) + output = Vector{UInt8}(undef, Int(expected)) + try + _decompress!(codec, output, source) + catch err + _codecfailure(err, codec) + end + return output + finally + iszero(copycharge) || _release!(something(budget), copycharge) + end +end + +function _checklevel(codec::Metadata.CompressionCodec.T, level::Integer, range::UnitRange{Int}) + level in range || throw(ArgumentError("$(codecname(codec)) compression level must be in $range, got $level")) + return Int(level) +end + +function _encoder(codec::Metadata.CompressionCodec.T, level::Union{Nothing,Integer}) + if codec == Metadata.CompressionCodec.SNAPPY + level === nothing || throw(ArgumentError("SNAPPY has no compression level")) + return ChunkCodecLibSnappy.SnappyEncodeOptions() + elseif codec == Metadata.CompressionCodec.GZIP + return ChunkCodecLibZlib.GzipEncodeOptions(; level=_checklevel(codec, something(level, 6), 0:9)) + elseif codec == Metadata.CompressionCodec.ZSTD + return ChunkCodecLibZstd.ZstdEncodeOptions(; compressionLevel=_checklevel(codec, something(level, 3), -131072:22)) + elseif codec == Metadata.CompressionCodec.BROTLI + return ChunkCodecLibBrotli.BrotliEncodeOptions(; quality=_checklevel(codec, something(level, 8), 0:11)) + end + return ChunkCodecLibLz4.LZ4BlockEncodeOptions(; compressionLevel=_checklevel(codec, something(level, 0), 0:12)) +end + +""" + compress(codec, bytes; level) -> Vector{UInt8} + +Compress one page payload. `level` is codec specific (GZIP 0-9, ZSTD -131072..22, BROTLI +0-11, LZ4_RAW 0-12); SNAPPY takes none. The deprecated LZ4 codec and LZO are not writable. +""" +function compress(codec::Metadata.CompressionCodec.T, bytes::AbstractVector{UInt8}; + level::Union{Nothing,Integer}=nothing) + length(bytes) <= typemax(Int32) || throw(ArgumentError("page payload exceeds 2^31 - 1 bytes")) + codecwritable(codec) || _unwritablecodec(codec) + if codec == Metadata.CompressionCodec.UNCOMPRESSED + level === nothing || throw(ArgumentError("UNCOMPRESSED has no compression level")) + return Vector{UInt8}(bytes) + end + return ChunkCodecCore.encode(_encoder(codec, level), _contiguous(bytes)) +end diff --git a/src/column.jl b/src/column.jl new file mode 100644 index 0000000..0121c17 --- /dev/null +++ b/src/column.jl @@ -0,0 +1,793 @@ +# Flat column chunk decoding: V1/V2 data pages, levels, and value encodings +# (Parquet 2.13.0 README "Data Pages", "Nulls", and "Column chunks"). + +function _physicaleltype(type::Metadata.Type.T) + type == Metadata.Type.BOOLEAN && return Bool + type == Metadata.Type.INT32 && return Int32 + type == Metadata.Type.INT64 && return Int64 + type == Metadata.Type.FLOAT && return Float32 + type == Metadata.Type.DOUBLE && return Float64 + type == Metadata.Type.BYTE_ARRAY && return Vector{UInt8} + type == Metadata.Type.FIXED_LEN_BYTE_ARRAY && return Vector{UInt8} + type == Metadata.Type.INT96 && throw(UnsupportedFeatureError( + "INT96 columns are not supported; the type is deprecated in Parquet")) + throw(FormatError("unknown physical type $type")) +end + +function _leafdensebytes(::Type{T}, count::Integer, + md::Metadata.ColumnMetaData) where {T} + bytes = _materializedarraybytes(T, count) + T === Vector{UInt8} || return bytes + payload = Int64(md.total_uncompressed_size) + payload >= 0 || throw(FormatError("negative column chunk uncompressed size")) + bytes = _materializedsum(bytes, + _materializedproduct(count, _MATERIALIZED_ARRAY_HEADER_BYTES)) + return _materializedsum(bytes, payload) +end + +function _leaflevelbytes(count::Integer) + bytes = _materializedarraybytes(UInt64, count) + return _materializedsum(bytes, _materializedarraybytes(UInt64, count)) +end + +function _leafchildbytes(::Type{T}, values::Vector{T}) where {T} + return Int64(0) +end + +function _leafchildbytes(::Type{Vector{UInt8}}, values::Vector{Vector{UInt8}}) + bytes = Int64(0) + for value in values + bytes = _materializedsum(bytes, + _materializedarraybytes(UInt8, length(value))) + end + return bytes +end + +function _leafretainedbytes(::Type{T}, entries::Integer, + values::Vector{T}) where {T} + bytes = _materializedsum(_leaflevelbytes(entries), + _materializedarraybytes(T, entries)) + return _materializedsum(bytes, _leafchildbytes(T, values)) +end + +function _pageentrycount(frame::PageFrame) + kind = pagekind(frame) + kind === :data_v1 && return Int64(frame.header.data_page_header.num_values) + kind === :data_v2 && return Int64(frame.header.data_page_header_v2.num_values) + kind === :dictionary && + return Int64(frame.header.dictionary_page_header.num_values) + return Int64(0) +end + +function _validatedpageentrycount(frame::PageFrame, limits::Limits) + entries = _pageentrycount(frame) + entries >= 0 || throw(FormatError("negative page value count")) + if pagekind(frame) === :dictionary + _checkdictionarypageencoding( + frame.header.dictionary_page_header.encoding) + _checklimit(:container_elements, entries, + limits.max_container_elements) + end + return entries +end + +function _pageworkingbytes(::Type{T}, frame::PageFrame) where {T} + count = _pageentrycount(frame) + count >= 0 || throw(FormatError("negative page value count")) + uncompressed = Int64(frame.header.uncompressed_page_size) + uncompressed >= 0 || throw(FormatError("negative uncompressed page size")) + bytes = _materializedproduct(4, _MATERIALIZED_OBJECT_BYTES) + bytes = _materializedsum(bytes, + _materializedarraybytes(UInt8, uncompressed)) + kind = pagekind(frame) + if kind in (:data_v1, :data_v2) + bytes = _materializedsum(bytes, _leaflevelbytes(count)) + bytes = _materializedsum(bytes, _materializedarraybytes(T, count)) + bytes = _materializedsum(bytes, + _materializedarraybytes(UInt64, count)) + elseif kind === :dictionary + bytes = _materializedsum(bytes, _materializedarraybytes(T, count)) + end + if T === Vector{UInt8} && kind in (:data_v1, :data_v2, :dictionary) + bytes = _materializedsum(bytes, + _materializedproduct(count, _MATERIALIZED_ARRAY_HEADER_BYTES)) + bytes = _materializedsum(bytes, uncompressed) + end + return bytes +end + +function _levelbitwidth(maxlevel::Integer) + maxlevel == 0 && return 0 + return 64 - leading_zeros(UInt64(maxlevel)) +end + +function _chunkmetadata(chunk::Metadata.ColumnChunk, node::SchemaNode) + chunk.file_path === nothing || throw(FormatError("column chunks stored in another file are not supported")) + chunk.crypto_metadata === nothing && chunk.encrypted_column_metadata === nothing || + throw(FormatError("encrypted column chunks are not supported")) + md = chunk.meta_data + md === nothing && throw(FormatError("column chunk has no metadata")) + md.type_ == node.element.type_ || + throw(FormatError("column chunk type $(md.type_) does not match the schema type $(node.element.type_)")) + md.path_in_schema == node.path || + throw(FormatError("column chunk path $(md.path_in_schema) does not match the schema path $(node.path)")) + md.num_values >= 0 || throw(FormatError("negative column chunk value count")) + md.total_compressed_size >= 0 || throw(FormatError("negative column chunk size")) + md.total_uncompressed_size >= 0 || + throw(FormatError("negative column chunk uncompressed size")) + return md +end + +# The chunk starts at its dictionary page, or at the earliest data/index page. Only the +# dictionary offset retains the historical zero sentinel. +function _chunkstart(md::Metadata.ColumnMetaData) + data = Int64(md.data_page_offset) + data >= 0 || throw(FormatError("negative data page offset $data")) + dictionary = md.dictionary_page_offset + dictionary === nothing || dictionary >= 0 || throw(FormatError( + "negative dictionary page offset $dictionary")) + index = md.index_page_offset + index === nothing || index >= 4 || throw(FormatError( + "index page offset $index is inside the file header")) + if dictionary !== nothing && dictionary > 0 + dictionary >= 4 || throw(FormatError( + "dictionary page offset $dictionary is inside the file header")) + data == 0 || data >= dictionary || throw(FormatError( + "data page offset $data precedes the dictionary page offset $dictionary")) + index === nothing || index >= dictionary || throw(FormatError( + "index page offset $index precedes the dictionary page")) + return Int64(dictionary) + end + index === nothing && return data + data > 0 || throw(FormatError( + "index page offset is present for a chunk with no data page")) + return min(data, Int64(index)) +end + +function _chunkrange(md::Metadata.ColumnMetaData, footeroffset::Int64) + footeroffset >= 0 || throw(FormatError("negative footer offset $footeroffset")) + start = _chunkstart(md) + size = Int64(md.total_compressed_size) + size >= 0 || throw(FormatError("negative column chunk size $size")) + if size == 0 + iszero(md.data_page_offset) || throw(FormatError( + "data page offset is present for an empty column chunk")) + dictionary = md.dictionary_page_offset + (dictionary === nothing || dictionary == 0) || throw(FormatError( + "dictionary page offset is present for an empty column chunk")) + md.index_page_offset === nothing || throw(FormatError( + "index page offset is present for an empty column chunk")) + return start, start + end + start >= 4 || throw(FormatError("column chunk offset $start is inside the file header")) + size <= footeroffset - start || throw(FormatError("column chunk extends past the footer at $footeroffset")) + stop = start + size + data = Int64(md.data_page_offset) + data == 0 || data < stop || throw(FormatError( + "data page offset $(md.data_page_offset) is outside the column chunk")) + index = md.index_page_offset + index === nothing || index < stop || throw(FormatError( + "index page offset $index is outside the column chunk")) + return start, stop +end + +function _chunkpageoffsetstate(md::Metadata.ColumnMetaData, position::Int64, + kind::Symbol, dictionaryseen::Bool, dataseen::Bool, + indexseen::Bool) + dictionary = md.dictionary_page_offset + realdictionary = dictionary !== nothing && dictionary > 0 ? + Int64(dictionary) : nothing + isdata = kind === :data_v1 || kind === :data_v2 + data = Int64(md.data_page_offset) + legacydictionary = realdictionary === nothing && + kind === :dictionary && !dictionaryseen && !dataseen && + data > 0 && position == data && position == _chunkstart(md) + if realdictionary !== nothing && position == realdictionary + kind === :dictionary || throw(FormatError( + "dictionary_page_offset does not point to a DICTIONARY_PAGE")) + end + if kind === :dictionary + (realdictionary !== nothing && position == realdictionary) || + legacydictionary || + throw(FormatError( + "dictionary page does not match dictionary_page_offset")) + (!dictionaryseen && !dataseen) || throw(FormatError( + "dictionary page is not the first physical page")) + dictionaryseen = true + end + if data > 0 && position == data + (isdata || legacydictionary) || throw(FormatError( + "data_page_offset does not point to a data page")) + end + if isdata && !dataseen + legacydata = realdictionary === nothing && dictionaryseen && + position > data + (data > 0 && position == data) || legacydata || throw(FormatError( + "first data page does not match data_page_offset")) + dataseen = true + end + index = md.index_page_offset + if index !== nothing && position == index + kind === :index || throw(FormatError( + "index_page_offset does not point to an INDEX_PAGE")) + indexseen = true + end + return dictionaryseen, dataseen, indexseen +end + +function _validatechunkpageoffsets(md::Metadata.ColumnMetaData, + dictionaryseen::Bool, dataseen::Bool, indexseen::Bool) + dictionary = md.dictionary_page_offset + (dictionary === nothing || dictionary == 0 || dictionaryseen) || + throw(FormatError( + "dictionary_page_offset does not identify a page frame")) + (iszero(md.data_page_offset) || dataseen) || throw(FormatError( + "data_page_offset does not identify a data page frame")) + (md.index_page_offset === nothing || indexseen) || throw(FormatError( + "index_page_offset does not identify a page frame")) + return +end + +function _fixedwidth(node::SchemaNode) + node.element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY || return nothing + return Int(node.element.type_length) +end + +function _decodevalues(::Type{T}, bytes::AbstractVector{UInt8}, count::Int, ::Nothing, + offset::Int, limits::Limits) where {T<:Union{Bool,Int32,Int64,Float32,Float64}} + return decode_plain(T, bytes, count; offset=offset, limits=limits) +end + +function _decodevalues(::Type{Vector{UInt8}}, bytes::AbstractVector{UInt8}, count::Int, ::Nothing, + offset::Int, limits::Limits) + return decode_plain_byte_array(bytes, count; offset=offset, limits=limits) +end + +function _decodevalues(::Type{Vector{UInt8}}, bytes::AbstractVector{UInt8}, count::Int, width::Int, + offset::Int, limits::Limits) + matrix, position = decode_plain_fixed(bytes, count, width; offset=offset, limits=limits) + values = Vector{Vector{UInt8}}(undef, count) + for index in 1:count + values[index] = matrix[:, index] + end + return values, position +end + +function _validatelevels(levels::Vector{UInt64}, maxlevel::Int, name::String) + atmaximum = 0 + for level in levels + level <= maxlevel || throw(FormatError("$name level $level exceeds the maximum $maxlevel")) + level == maxlevel && (atmaximum += 1) + end + return atmaximum +end + +function _matrixvaluescharged(matrix::Matrix{UInt8}, + budget::_LiveByteBudget=_LiveByteBudget(Limits())) + charge = _materializedarraybytes(Vector{UInt8}, size(matrix, 2)) + charge = _materializedsum(charge, _materializedproduct(size(matrix, 2), + _materializedarraybytes(UInt8, size(matrix, 1)))) + _reserve!(budget, charge) + values = Vector{Vector{UInt8}}(undef, size(matrix, 2)) + try + for index in eachindex(values) + values[index] = matrix[:, index] + end + catch + _release!(budget, charge) + rethrow() + end + return values, charge +end + +function _matrixvalues(matrix::Matrix{UInt8}, + budget::_LiveByteBudget=_LiveByteBudget(Limits())) + values, _ = _matrixvaluescharged(matrix, budget) + return values +end + +function _decodebooleanrle(bytes::AbstractVector{UInt8}, count::Int, offset::Int, + limits::Limits) + encoded, position = decode_hybrid(bytes, count, 1; offset=offset, length_prefix=true, + limits=limits) + values = Vector{Bool}(undef, count) + for index in eachindex(encoded) + encoded[index] <= 1 || throw(FormatError("RLE Boolean value $(encoded[index]) exceeds 1")) + values[index] = !iszero(encoded[index]) + end + return values, position +end + +function _encodingerror(encoding::Metadata.Encoding.T, type) + throw(FormatError("data page encoding $encoding is not valid for physical type $type")) +end + +function _dictionaryselection(dictionary, indices::Vector{UInt64}) + count = UInt64(length(dictionary.values)) + for raw in indices + raw < count || throw(FormatError( + "dictionary index $raw is outside a dictionary of $count entries")) + end + return Int64(0) +end + +function _dictionaryselection(dictionary::DecodedDictionary{Vector{UInt8}}, + indices::Vector{UInt64}) + count = UInt64(length(dictionary.values)) + bytes = Int64(0) + for raw in indices + raw < count || throw(FormatError( + "dictionary index $raw is outside a dictionary of $count entries")) + bytes = _materializedsum(bytes, + length(dictionary.values[Int(raw) + 1])) + end + return bytes +end + +function _decodedictionaryvaluesbudgeted(dictionary, bytes::AbstractVector{UInt8}, + count::Int, offset::Int, allowance::Integer, limits::Limits, + budget::_LiveByteBudget) + dictionary === nothing && throw(FormatError( + "dictionary-encoded data page has no dictionary page")) + indices, position = _decodedictionaryindices(bytes, count, offset, limits) + selected = _dictionaryselection(dictionary, indices) + expansion = max(Int64(0), selected - Int64(allowance)) + iszero(expansion) || _reserve!(budget, expansion) + output = try + _lookupdictionary(dictionary, indices) + catch + iszero(expansion) || _release!(budget, expansion) + rethrow() + end + return output, position +end + +function _decodeencodedvalues(::Type{T}, encoding::Metadata.Encoding.T, + bytes::AbstractVector{UInt8}, count::Int, width, offset::Int, limits::Limits, + budget::_LiveByteBudget) where {T} + if encoding == Metadata.Encoding.PLAIN + values, position = _decodevalues(T, bytes, count, width, offset, limits) + return values, position, Int64(0) + end + _isdictionaryencoding(encoding) && return nothing + if encoding == Metadata.Encoding.DELTA_BINARY_PACKED + T <: Union{Int32,Int64} || return _encodingerror(encoding, T) + return _decode_delta_binary_packed(T, bytes, count; offset=offset, + limits=limits, budget=budget) + elseif encoding == Metadata.Encoding.DELTA_LENGTH_BYTE_ARRAY + T == Vector{UInt8} && width === nothing || return _encodingerror(encoding, T) + return _decode_delta_length_byte_array(bytes, count; offset=offset, + limits=limits, budget=budget) + elseif encoding == Metadata.Encoding.DELTA_BYTE_ARRAY + T == Vector{UInt8} || return _encodingerror(encoding, T) + if width === nothing + return _decode_delta_byte_array(bytes, count; offset=offset, + limits=limits, budget=budget) + end + matrix, position, matrixcharge = _decode_delta_byte_array_fixed(bytes, + count, width; + offset=offset, limits=limits, budget=budget) + values, valuescharge = try + _matrixvaluescharged(matrix, budget) + catch + _release!(budget, matrixcharge) + rethrow() + end + _release!(budget, matrixcharge) + return values, position, valuescharge + elseif encoding == Metadata.Encoding.BYTE_STREAM_SPLIT + if T <: Union{Int32,Int64,Float32,Float64} + values, position = decode_byte_stream_split(T, bytes, count; + offset=offset, limits=limits) + return values, position, Int64(0) + elseif T == Vector{UInt8} && width !== nothing + matrix, position = decode_byte_stream_split_fixed(bytes, count, width; + offset=offset, limits=limits) + values, valuescharge = _matrixvaluescharged(matrix, budget) + return values, position, valuescharge + end + return _encodingerror(encoding, T) + elseif encoding == Metadata.Encoding.RLE + T == Bool || return _encodingerror(encoding, T) + values, position = _decodebooleanrle(bytes, count, offset, limits) + return values, position, Int64(0) + end + throw(FormatError("data page encoding $encoding is not supported")) +end + +function _decodelevelv1(bytes::AbstractVector{UInt8}, count::Int, + encoding::Metadata.Encoding.T, maxlevel::Int, offset::Int, name::String, + limits::Limits) + maxlevel == 0 && return zeros(UInt64, count), offset + if encoding == Metadata.Encoding.RLE + levels, position = decode_hybrid(bytes, count, _levelbitwidth(maxlevel); offset=offset, + length_prefix=true, limits=limits) + elseif encoding == Metadata.Encoding.BIT_PACKED + levels, position = decode_bit_packed(bytes, count, _levelbitwidth(maxlevel); + offset=offset, limits=limits) + else + throw(FormatError("$name level encoding $encoding is not supported")) + end + _validatelevels(levels, maxlevel, name) + return levels, position +end + +function _decodelevelsv1(bytes::AbstractVector{UInt8}, count::Int, + header::Metadata.DataPageHeader, node::SchemaNode, limits::Limits) + repetition, position = _decodelevelv1(bytes, count, header.repetition_level_encoding, + Int(node.max_repetition_level), 1, "repetition", limits) + definition, position = _decodelevelv1(bytes, count, header.definition_level_encoding, + Int(node.max_definition_level), position, "definition", limits) + present = _validatelevels(definition, Int(node.max_definition_level), "definition") + return repetition, definition, present, position +end + +function _decodelevelsv2(bytes::AbstractVector{UInt8}, count::Int, maxlevel::Int, + name::String, limits::Limits) + isempty(bytes) && maxlevel == 0 && return zeros(UInt64, count) + levels, position = decode_hybrid(bytes, count, _levelbitwidth(maxlevel); + offset=1, limits=limits) + position == length(bytes) + 1 || + throw(FormatError("data page V2 $name levels have trailing bytes")) + _validatelevels(levels, maxlevel, name) + return levels +end + +function _decodedatapage(::Type{T}, frame::PageFrame, md::Metadata.ColumnMetaData, + node::SchemaNode, dictionary, remaining::Int, limits::Limits, + budget::_LiveByteBudget) where {T} + header = frame.header.data_page_header + count = Int(header.num_values) + count >= 0 || throw(FormatError("negative data page value count")) + count <= remaining || + throw(FormatError("data pages carry more values than declared by the column chunk")) + bytes = decompresspage(frame, md.codec; limits=limits, budget=budget) + repetition, definition, present, position = _decodelevelsv1(bytes, count, header, + node, limits) + encoding = header.encoding + decoded = _decodeencodedvalues(T, encoding, bytes, present, _fixedwidth(node), + position, limits, budget) + if decoded === nothing + values, position = _decodedictionaryvaluesbudgeted(dictionary, bytes, + present, position, frame.header.uncompressed_page_size, limits, budget) + else + values, position, _ = decoded + end + position == length(bytes) + 1 || + throw(FormatError("data page has $(length(bytes) - position + 1) bytes after its values")) + return repetition, definition, values +end + +function _v2values(frame::PageFrame, codec::Metadata.CompressionCodec.T, + limits::Limits, budget::_LiveByteBudget) + header = frame.header + data = header.data_page_header_v2 + repetition = Int64(data.repetition_levels_byte_length) + definition = Int64(data.definition_levels_byte_length) + repetition >= 0 || throw(FormatError("negative data page V2 repetition-level byte length")) + definition >= 0 || throw(FormatError("negative data page V2 definition-level byte length")) + levels = repetition + definition + compressed = Int64(header.compressed_page_size) + uncompressed = Int64(header.uncompressed_page_size) + levels <= compressed || throw(FormatError("data page V2 levels exceed its compressed size")) + levels <= uncompressed || throw(FormatError("data page V2 levels exceed its uncompressed size")) + repetition <= typemax(Int) && definition <= typemax(Int) && levels <= typemax(Int) || + throw(FormatError("data page V2 level lengths overflow Int")) + repetitionlength = Int(repetition) + definitionlength = Int(definition) + levellength = Int(levels) + repetitionbytes = @view frame.payload[1:repetitionlength] + definitionbytes = @view frame.payload[(repetitionlength + 1):levellength] + encoded = @view frame.payload[(levellength + 1):end] + expected = Int(uncompressed - levels) + actual = Int(compressed - levels) + length(encoded) == actual || throw(FormatError("data page V2 value-section size is inconsistent")) + compressedvalues = something(data.is_compressed, true) + values = isempty(encoded) && expected == 0 ? UInt8[] : + decompress(compressedvalues ? codec : Metadata.CompressionCodec.UNCOMPRESSED, + encoded, expected; limits=limits, budget=budget) + return repetitionbytes, definitionbytes, values +end + +function _decodedatapagev2(::Type{T}, frame::PageFrame, md::Metadata.ColumnMetaData, + node::SchemaNode, dictionary, remaining::Int, limits::Limits, + budget::_LiveByteBudget) where {T} + header = frame.header.data_page_header_v2 + count = Int(header.num_values) + nulls = Int(header.num_nulls) + rows = Int(header.num_rows) + count >= 0 || throw(FormatError("negative data page V2 value count")) + 0 <= nulls <= count || throw(FormatError("invalid data page V2 null count $nulls for $count values")) + rows >= 0 || throw(FormatError("negative data page V2 row count")) + count <= remaining || + throw(FormatError("data pages carry more values than declared by the column chunk")) + repetitionbytes, definitionbytes, bytes = _v2values(frame, md.codec, + limits, budget) + repetition = _decodelevelsv2(repetitionbytes, count, + Int(node.max_repetition_level), "repetition", limits) + definition = _decodelevelsv2(definitionbytes, count, + Int(node.max_definition_level), "definition", limits) + isempty(repetition) || iszero(first(repetition)) || + throw(FormatError("data page V2 starts with repetition level $(first(repetition))")) + actualrows = Base.count(iszero, repetition) + rows == actualrows || + throw(FormatError("data page V2 declares $rows rows but its repetition levels contain $actualrows")) + present = _validatelevels(definition, Int(node.max_definition_level), "definition") + count - present == nulls || + throw(FormatError("data page V2 declares $nulls nulls but its definition levels contain $(count - present)")) + decoded = _decodeencodedvalues(T, header.encoding, bytes, present, + _fixedwidth(node), 1, limits, budget) + if decoded === nothing + values, position = _decodedictionaryvaluesbudgeted(dictionary, bytes, + present, 1, frame.header.uncompressed_page_size, limits, budget) + else + values, position, _ = decoded + end + position == length(bytes) + 1 || + throw(FormatError("data page V2 has $(length(bytes) - position + 1) bytes after its values")) + return repetition, definition, values +end + +function _appendleafpage!(repetition::Vector{UInt64}, definition::Vector{UInt64}, + values::Vector{T}, produced::Int, page) where {T} + pagerepetition, pagedefinition, pagevalues = page + entries = length(pagerepetition) + length(pagedefinition) == entries || + throw(FormatError("data page repetition and definition counts differ")) + copyto!(repetition, produced + 1, pagerepetition, 1, entries) + copyto!(definition, produced + 1, pagedefinition, 1, entries) + append!(values, pagevalues) + return produced + entries +end + +function _leafoperationcharge(budget::_LiveByteBudget, start::Int64) + used = _budgetused(budget) + used >= start || throw(AssertionError( + "leaf decoding released bytes owned by its caller")) + return used - start +end + +function _transferleafcharge!(budget::_LiveByteBudget, start::Int64, + releasing::Int64, retained::Int64) + charge = _leafoperationcharge(budget, start) + releasing <= charge || throw(AssertionError( + "leaf decoding releases more bytes than it owns")) + available = charge - releasing + available >= retained || _reserve!(budget, retained - available) + return +end + +function _readleafstream(::Type{T}, src::AbstractSource, md::Metadata.ColumnMetaData, + node::SchemaNode, start::Int64, stop::Int64, limits::Limits, + budget::_LiveByteBudget; + expected_rows=nothing) where {T} + operationstart = _budgetused(budget) + try + count = Int(md.num_values) + _reserve!(budget, _leaflevelbytes(count)) + _reserve!(budget, _leafdensebytes(T, count, md)) + repetition = Vector{UInt64}(undef, count) + definition = Vector{UInt64}(undef, count) + values = T[] + sizehint!(values, count) + retainedbase = _materializedsum(_leaflevelbytes(count), + _materializedarraybytes(T, count)) + childbytes = Int64(0) + position = start + framecount = Int64(0) + produced = 0 + dictionary = nothing + dictionarycharge = Int64(0) + seendata = false + indexseen = false + while position < stop + frame = readpage(src, position, stop, limits; budget=budget) + frameend = position + workingcharge = Int64(0) + try + frameend = pageend(frame) + frameend > position || throw(FormatError( + "column chunk contains a nonadvancing page frame")) + frameend <= stop || throw(FormatError( + "page frame extends past the column chunk")) + kind = pagekind(frame) + dictionaryseen = dictionary !== nothing + dictionaryseen, seendata, indexseen = + _chunkpageoffsetstate(md, position, kind, + dictionaryseen, seendata, indexseen) + entries = _validatedpageentrycount(frame, limits) + if kind === :data_v1 || kind === :data_v2 + entries <= count - produced || throw(FormatError( + "data pages carry more values than declared by the column chunk")) + end + framecount = _nextpageframecount(framecount, limits) + requested = _pageworkingbytes(T, frame) + _reserve!(budget, requested) + workingcharge = requested + if kind === :data_v1 + page = _decodedatapage(T, frame, md, node, dictionary, + count - produced, limits, budget) + childbytes = _materializedsum(childbytes, + _leafchildbytes(T, page[3])) + produced = _appendleafpage!(repetition, definition, values, + produced, page) + retained = _materializedsum(retainedbase, childbytes) + releasing = _materializedsum(workingcharge, + frame.materializedcharge) + _transferleafcharge!(budget, operationstart, releasing, + retained) + seendata = true + elseif kind === :data_v2 + page = _decodedatapagev2(T, frame, md, node, dictionary, + count - produced, limits, budget) + childbytes = _materializedsum(childbytes, + _leafchildbytes(T, page[3])) + produced = _appendleafpage!(repetition, definition, values, + produced, page) + retained = _materializedsum(retainedbase, childbytes) + releasing = _materializedsum(workingcharge, + frame.materializedcharge) + _transferleafcharge!(budget, operationstart, releasing, + retained) + seendata = true + elseif kind === :dictionary + seendata && throw(FormatError( + "dictionary page follows a data page in the column chunk")) + dictionary === nothing || throw(FormatError( + "column chunk has more than one dictionary page")) + dictionary = _decodedictionarypage(T, frame, md, node, + limits, budget) + dictionarycharge = workingcharge + workingcharge = Int64(0) + end + finally + iszero(workingcharge) || _release!(budget, workingcharge) + _release!(budget, frame.materializedcharge) + end + position = frameend + end + position == stop || throw(FormatError( + "column chunk page walk does not end at its declared boundary")) + _validatechunkpageoffsets(md, dictionary !== nothing, seendata, + indexseen) + produced == count || throw(FormatError( + "column chunk produced $produced of the $count declared values")) + stream = LeafStream(repetition, definition, values, + Int(node.max_repetition_level), Int(node.max_definition_level); + expected_rows=expected_rows) + retained = _leafretainedbytes(T, count, values) + retained == _materializedsum(retainedbase, childbytes) || + throw(AssertionError("leaf retained-byte accounting is inconsistent")) + if !iszero(dictionarycharge) + _transferleafcharge!(budget, operationstart, dictionarycharge, + retained) + _release!(budget, dictionarycharge) + end + charge = _leafoperationcharge(budget, operationstart) + charge >= retained || throw(AssertionError( + "leaf decoding did not retain enough bytes for its result")) + charge == retained || _release!(budget, charge - retained) + return stream + catch + charge = _leafoperationcharge(budget, operationstart) + iszero(charge) || _release!(budget, charge) + rethrow() + end +end + +function _flatcolumn(stream::LeafStream{T}, node::SchemaNode, + budget::_LiveByteBudget) where {T} + node.max_repetition_level == 0 || + throw(FormatError("repeated columns are not supported yet")) + node.max_definition_level == 0 && return stream.values + outputcharge = _reservearray!(budget, Union{Missing,T}, length(stream)) + output = Vector{Union{Missing,T}}(undef, length(stream)) + try + nextvalue = 1 + maxdefinition = UInt64(node.max_definition_level) + for index in eachindex(stream.definition) + if stream.definition[index] == maxdefinition + output[index] = stream.values[nextvalue] + nextvalue += 1 + else + output[index] = missing + end + end + catch + _release!(budget, outputcharge) + rethrow() + end + return output +end + +function _flatcolumn(stream::LeafStream{T}, node::SchemaNode) where {T} + return _flatcolumn(stream, node, _LiveByteBudget(Limits())) +end + +function _flattenleafstream(stream::LeafStream, node::SchemaNode, + budget::_LiveByteBudget) + output = try + _flatcolumn(stream, node, budget) + catch + _release!(budget, _leafretainedbytes(eltype(stream.values), + length(stream), stream.values)) + rethrow() + end + _release!(budget, _leaflevelbytes(length(stream))) + if output !== stream.values + _release!(budget, _materializedarraybytes(eltype(stream.values), + length(stream))) + end + return output +end + +""" + readleafstream(src, chunk, node, footeroffset; expected_rows, limits) -> LeafStream + +Decode one physical column chunk into repetition levels, definition levels, and dense +present values. `expected_rows` optionally validates the number of zero repetition levels. +""" +function readleafstream(src::AbstractSource, chunk::Metadata.ColumnChunk, + node::SchemaNode, footeroffset::Integer; expected_rows=nothing, + limits::Limits=Limits(), budget::_LiveByteBudget=_LiveByteBudget(limits)) + md = _chunkmetadata(chunk, node) + T = _physicaleltype(md.type_) + md.num_values <= typemax(Int) || throw(FormatError("column chunk value count overflows")) + _checklimit(:container_elements, md.num_values, limits.max_container_elements) + typemin(Int64) <= footeroffset <= typemax(Int64) || throw(ArgumentError( + "footer offset does not fit Int64")) + footer = Int64(footeroffset) + footer <= _checkedsourcelength(src) || + throw(FormatError("footer offset is past the end of the source")) + start, stop = _chunkrange(md, footer) + return _readleafstream(T, src, md, node, start, stop, limits, budget; + expected_rows=expected_rows) +end + +""" + readcolumn(src, chunk, node, footeroffset; limits) -> Vector + +Decode a flat (max repetition level 0) column chunk into a concrete +vector. Optional columns yield `Vector{Union{Missing,T}}`; required columns yield `Vector{T}`. +BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY values are `Vector{UInt8}` at this physical layer. +""" +function readcolumn(src::AbstractSource, chunk::Metadata.ColumnChunk, node::SchemaNode, footeroffset::Integer; + limits::Limits=Limits(), budget::_LiveByteBudget=_LiveByteBudget(limits)) + node.max_repetition_level == 0 || + throw(FormatError("repeated columns are not supported yet")) + stream = readleafstream(src, chunk, node, footeroffset; limits=limits, + budget=budget) + return _flattenleafstream(stream, node, budget) +end + +function _columnselection(metadata::Metadata.FileMetaData, schema::Schema, + rowgroup::Integer, column::Integer) + 1 <= rowgroup <= length(metadata.row_groups) || + throw(ArgumentError("row group $rowgroup is out of range")) + columns = metadata.row_groups[rowgroup].columns + 1 <= column <= length(schema.leaves) || + throw(ArgumentError("column $column is out of range")) + length(columns) == length(schema.leaves) || + throw(FormatError("row group $rowgroup has $(length(columns)) column chunks for $(length(schema.leaves)) leaves")) + return columns[column], schema.leaves[column] +end + +function readleafstream(file::File, metadata::Metadata.FileMetaData, schema::Schema, + rowgroup::Integer, column::Integer; expected_rows=nothing, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + chunk, node = _columnselection(metadata, schema, rowgroup, column) + return readleafstream(file.source, chunk, node, file.footer.offset; + expected_rows=expected_rows, limits=limits, budget=budget) +end + +function readcolumn(file::File, metadata::Metadata.FileMetaData, schema::Schema, rowgroup::Integer, + column::Integer; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + chunk, node = _columnselection(metadata, schema, rowgroup, column) + node.max_repetition_level == 0 || + throw(FormatError("repeated columns are not supported yet")) + rows = metadata.row_groups[rowgroup].num_rows + stream = readleafstream(file.source, chunk, node, file.footer.offset; + expected_rows=rows, limits=limits, budget=budget) + return _flattenleafstream(stream, node, budget) +end diff --git a/src/cursor.jl b/src/cursor.jl deleted file mode 100644 index 3910587..0000000 --- a/src/cursor.jl +++ /dev/null @@ -1,509 +0,0 @@ -## -# layer 3 access -# read data as records which are named tuple representations of the schema - -## -# Column cursor iterates through all values of the column, including null values. -# Each iteration returns the value (as a Union{T,Nothing}), definition level, and repetition level for each value. -# Row can be deduced from repetition level. -mutable struct ColCursor{T} - par::Parquet.File - colname::Vector{String} # column name (full path in schema) - colnamesym::Vector{Symbol} # column name converted to symbols - required_at::Vector{Bool} # at what level in the schema the column is required - repeated_at::Vector{Bool} # at what level in the schema the column is repeated - logical_converter_fn::Function # column converter function as per schema or options (identity if none) - maxdefn::Int32 # maximum definition level of the column as per schema - - row::Int # current row - rows::UnitRange{Int} # row range to limit to - rowgroups::Vector{RowGroup} # list of row groups - row_positions::Vector{Int64} # starting positions if each rowgroup - rgidx::Int # current rowgroup - colchunks::Union{Vector{ColumnChunk},Nothing} # list of column chunk in the current rowgroup (metadata only) - ccidx::Int # index of column chunk that contains the current position - - pageiter::Union{Nothing,ColumnChunkPageValues{T}} # iterator for the current page - pagerange::Union{UnitRange{Int64},Nothing} # range of positions (rows) that the current page contains - pageiternext::Int64 - pagevals::OutputState{T} - page_defn::OutputState{Int32} - page_repn::OutputState{Int32} - - valpos::Int64 # current position within values of the current page - levelpos::Int64 # current position within levels of the current page - levelend::Int64 - - function ColCursor{T}(par::Parquet.File, row_positions::Vector{Int64}, colname::Vector{String}, rows::UnitRange) where T - colnamesym = [Symbol(name) for name in colname] - sch = schema(par) - - len_colname_parts = length(colname) - required_at = Array{Bool}(undef, len_colname_parts) - repeated_at = Array{Bool}(undef, len_colname_parts) - for idx in 1:len_colname_parts - partname = colname[1:idx] - required_at[idx] = isrequired(sch, partname) # determine whether field is optional and repeated - repeated_at[idx] = isrepeated(sch, partname) - end - - logical_converter_fn = logical_converter(sch, colname) - maxdefn = max_definition_level(sch, colname) - new{T}(par, colname, colnamesym, required_at, repeated_at, logical_converter_fn, maxdefn, 0, rows, rowgroups(par), row_positions, 0, nothing, 0, nothing, nothing, 0) - end -end - -function ColCursor(par::Parquet.File, colname::Vector{String}; rows::UnitRange=1:nrows(par), row::Signed=first(rows)) - row_positions = rowgroup_row_positions(par) - @assert last(rows) <= nrows(par) - @assert first(rows) >= 1 - T = elemtype(schema(par), colname) - cursor = ColCursor{T}(par, row_positions, colname, rows) - setrow(cursor, Int64(row)) - cursor -end - -eltype(::Type{ColCursor{T}}) where {T} = NamedTuple{(:value, :defn_level, :repn_level),Tuple{Union{Nothing,T},Int64,Int64}} -length(cursor::ColCursor) = length(cursor.rows) - -function setrow(cursor::ColCursor{T}, row::Int64) where {T} - # check if cursor is done - if ((cursor.row > 0) && !(cursor.row in cursor.rows)) || isempty(cursor.rowgroups) - cursor.colchunks = nothing - cursor.ccidx = 0 - cursor.valpos = cursor.levelpos = 0 - cursor.levelend = -1 - return - end - - # check if we are already on that row - (cursor.row == row) && return - - par = cursor.par - prevrow = cursor.row - cursor.row = row - - # advance row group if needed - if (cursor.rgidx == 0) || (cursor.row_positions[cursor.rgidx+1] <= row) - rgidx = findfirst(x->x>row, cursor.row_positions) - if rgidx === nothing - cursor.rgidx = length(cursor.rowgroups) + 1 - return - else - cursor.rgidx = rgidx - 1 - rg = cursor.rowgroups[cursor.rgidx] - cursor.colchunks = columns(par, rg, cursor.colname) - cursor.pagerange = nothing - cursor.pageiternext = 0 - cursor.pageiter = nothing - cursor.ccidx = 0 - end - end - - # advance column chunk and page within column chunk if needed - while (cursor.pagerange === nothing) || !(row in cursor.pagerange) - cursor.valpos = cursor.levelpos = 0 - cursor.levelend = -1 - startrow = (cursor.pagerange === nothing) ? cursor.row_positions[cursor.rgidx] : (last(cursor.pagerange) + 1) - if cursor.pageiter === nothing - # need to start a page cursor for a new column chunk - cursor.ccidx += 1 - cursor.pageiter = ColumnChunkPageValues(par, cursor.colchunks[cursor.ccidx], T, cursor.logical_converter_fn) - cursor.pageiternext = 0 - end - page_iter_result = (cursor.pageiternext > 0) ? iterate(cursor.pageiter, cursor.pageiternext) : iterate(cursor.pageiter) - if page_iter_result === nothing - # need to start on a new column chunk - cursor.pageiter = nothing - else - resultdata,cursor.pageiternext = page_iter_result - cursor.pagevals = resultdata.value - cursor.page_repn = resultdata.repn_level - cursor.page_defn = resultdata.defn_level - nrowspage = 0 - if cursor.pageiter.has_repn_levels - for i in 1:cursor.page_repn.offset - (cursor.page_repn.data[i] !== Int32(0)) && (nrowspage += 1) - end - else - if cursor.pageiter.has_defn_levels - nrowspage = cursor.page_defn.offset - else - nrowspage = cursor.pagevals.offset # number of values is number of rows - end - end - if nrowspage > 0 - cursor.pagerange = startrow:(startrow+nrowspage-1) - end - end - end - - # advance value and level positions within the page - if cursor.pageiter.has_defn_levels - if cursor.levelpos == 0 - for idx in 1:(row - first(cursor.pagerange) + 1) - cursor.levelpos += 1 - (cursor.page_defn.data[cursor.levelpos] === cursor.maxdefn) && (cursor.valpos += 1) - end - elseif cursor.page_defn.data[cursor.levelpos] === cursor.maxdefn - cursor.valpos += 1 - end - else - # all entries are required, so there must be a corresponding value - cursor.valpos = cursor.levelpos = row - first(cursor.pagerange) + 1 - end - - if cursor.pageiter.has_repn_levels - # multiple entries may constitute one row - if cursor.levelend == -1 - cursor.levelend = Int64(something(findnext(x->x===Int32(0), cursor.page_repn.data, cursor.levelpos+1), cursor.page_repn.offset+1) - 1) - end - else - # no repetitions, so each entry corresponds to one full row - cursor.levelend = cursor.levelpos - end - nothing -end - -function _start(cursor::ColCursor) - setrow(cursor, Int64(first(cursor.rows))) - cursor.row, cursor.levelpos -end -function _done(cursor::ColCursor, rowandlevel::Tuple{Int64,Int64}) - row, levelpos = rowandlevel - (levelpos > cursor.levelend) || !(row in cursor.rows) -end -function _next(cursor::ColCursor{T}, rowandlevel::Tuple{Int64,Int64}) where {T} - # find values for current row and level in row - row, levelpos = rowandlevel - (levelpos == cursor.levelpos) || throw(InvalidStateException("Invalid column cursor state", :levelpos)) - - maxdefn = cursor.maxdefn - defn_level = cursor.pageiter.has_defn_levels ? cursor.page_defn.data[cursor.levelpos] : maxdefn - repn_level = cursor.pageiter.has_repn_levels ? cursor.page_repn.data[cursor.levelpos] : Int32(0) - if defn_level == maxdefn - val = (cursor.pagevals.data[cursor.valpos])::T - else - val = nothing - end - - # advance row - cursor.levelpos += 1 - if cursor.levelpos > cursor.levelend - row += 1 - cursor.levelend = -1 - setrow(cursor, Int64(row)) - end - - NamedTuple{(:value, :defn_level, :repn_level),Tuple{Union{Nothing,T},Int64,Int64}}((val, defn_level, repn_level)), (row, cursor.levelpos) -end - -function Base.iterate(cursor::ColCursor{T}, state) where {T} - _done(cursor, state) && return nothing - return _next(cursor, state) -end - -function Base.iterate(cursor::ColCursor) - r = iterate(cursor, _start(cursor)) - return r -end - -## - -mutable struct BatchedColumnsCursor{T} - par::Parquet.File - colnames::Vector{Vector{String}} - colcursors::Vector{ColCursor} - colstates::Vector{Tuple{Int64,Int64}} - colbuffs::Vector{Union{Nothing,Vector}} - batchid::Int - rows::UnitRange{Int64} - row::Int64 - batchsize::Int64 - nbatches::Int - reusebuffer::Bool - use_threads::Bool -end - -""" -Create cursor to iterate over batches of column values. Each iteration returns a named tuple of column names with batch of column values. Files with nested schemas can not be read with this cursor. - -```julia -BatchedColumnsCursor(par::Parquet.File; kwargs...) -``` - -Cursor options: -- `rows`: the row range to iterate through, all rows by default. -- `batchsize`: maximum number of rows to read in each batch (default: row count of first row group). -- `reusebuffer`: boolean to indicate whether to reuse the buffers with every iteration; if each iteration processes the batch and does not need to refer to the same data buffer again, then setting this to `true` reduces GC pressure and can help significantly while processing large files. -- `use_threads`: whether to use threads while reading the file; applicable only for Julia v1.3 and later and switched on by default if julia processes is started with multiple threads. -""" -function BatchedColumnsCursor(par::Parquet.File; - rows::UnitRange=1:nrows(par), - batchsize::Signed=length(rows) > 0 ? min(length(rows), first(rowgroups(par)).num_rows) : 0, - reusebuffer::Bool=false, - use_threads::Bool=(nthreads() > 1)) - - sch = schema(par) - - # supports only non nested columns as of now - if !all(num_children(schemaelem) == 0 for schemaelem in sch.schema[2:end]) - error("nested schemas are not supported with BatchedColumnsCursor yet") - end - - colcursors = [ColCursor(par, colname; rows=rows) for colname in colnames(par)] - rectype = ntcolstype(sch, sch.schema[1]) - nbatches = batchsize > 0 ? ceil(Int, length(rows)/batchsize) : 0 - colbuffs = Union{Nothing,Vector}[nothing for idx in 1:length(colcursors)] - - BatchedColumnsCursor{rectype}(par, colnames(par), colcursors, Array{Tuple{Int64,Int64}}(undef, length(colcursors)), colbuffs, 1, rows, first(rows), batchsize, nbatches, reusebuffer, (VERSION < v"1.3") ? false : use_threads) -end - -eltype(::Type{BatchedColumnsCursor{T}}) where {T} = T -length(cursor::BatchedColumnsCursor) = cursor.nbatches - -function colcursor_advance(colcursor::ColCursor, rows_by::Int64, vals_by::Int64=rows_by) - if (colcursor.row + rows_by) > last(colcursor.pagerange) - setrow(colcursor, Int64(colcursor.row+rows_by)) - else - colcursor.valpos += vals_by - colcursor.row += rows_by - colcursor.levelpos += rows_by - end - nothing -end - -function colcursor_values(colcursor::ColCursor{T}, batchsize::Int64, ::Type{Vector{Union{Missing,T}}}, cache) where {T} - row = colcursor.row - batchsize = min(batchsize, last(colcursor.rows)-row+1) - vals = (cache === nothing) ? Array{Union{Missing,T}}(undef, batchsize) : resize!(cache::Vector{Union{Missing,T}}, batchsize) - - fillpos = 1 - while fillpos <= batchsize - pagevals = colcursor.pagevals - defn_levels = colcursor.page_defn - val_idx = (colcursor.valpos == 0) ? 0 : (colcursor.valpos-1) - nvals_from_page = min(batchsize - fillpos + 1, defn_levels.offset - colcursor.levelpos + 1) - @inbounds for idx in 1:nvals_from_page - if defn_levels.data[colcursor.levelpos+idx-1] === Int32(1) - vals[fillpos+idx-1] = pagevals.data[val_idx+=1] - else - vals[fillpos+idx-1] = missing - end - end - fillpos += nvals_from_page - valposincr = (colcursor.valpos == 0) ? val_idx : (val_idx - colcursor.valpos + 1) - colcursor_advance(colcursor, nvals_from_page, Int64(valposincr)) - end - vals -end - -function colcursor_values(colcursor::ColCursor{T}, batchsize::Int64, ::Type{Vector{T}}, cache) where {T} - row = colcursor.row - batchsize = min(batchsize, last(colcursor.rows)-row+1) - vals = (cache === nothing) ? Array{T}(undef, batchsize) : resize!(cache::Vector{T}, batchsize) - - fillpos = 1 - while fillpos <= batchsize - pagevals = colcursor.pagevals - nvals_from_page = min(batchsize - fillpos + 1, pagevals.offset - colcursor.valpos + 1) - @inbounds for idx in 1:nvals_from_page - vals[fillpos+idx-1] = pagevals.data[pagevals.offset+idx] - end - fillpos += nvals_from_page - colcursor_advance(colcursor, nvals_from_page) - end - vals -end - -function Base.iterate(cursor::BatchedColumnsCursor{T}, batchid) where {T} - (batchid > length(cursor)) && (return nothing) - - colcursors = cursor.colcursors - coltypes = T.types - if cursor.use_threads - L = length(colcursors) - colvals = Array{Any}(undef, L) - @threads for idx in 1:L - colbuff = cursor.reusebuffer ? cursor.colbuffs[idx] : nothing - colvals[idx] = colcursor_values(colcursors[idx], cursor.batchsize, coltypes[idx], colbuff) - end - cursor.reusebuffer && (cursor.colbuffs = colvals) - else - if cursor.reusebuffer - colvals = cursor.colbuffs = [colcursor_values(colcursor,cursor.batchsize,coltype,colbuff) for (colcursor,coltype,colbuff) in zip(colcursors,coltypes,cursor.colbuffs)] - else - colvals = [colcursor_values(colcursor,cursor.batchsize,coltype,nothing) for (colcursor,coltype) in zip(colcursors,coltypes)] - end - end - - cursor.row += cursor.batchsize - cursor.batchid += 1 - T(colvals), cursor.batchid -end - -function Base.iterate(cursor::BatchedColumnsCursor{T}) where {T} - cursor.row = first(cursor.rows) - for colcursor in cursor.colcursors - setrow(colcursor, Int64(cursor.row)) - end - iterate(cursor, cursor.batchid) -end - - -## - -mutable struct RecordCursor{T} - par::Parquet.File - colnames::Vector{Vector{String}} - colcursors::Vector{ColCursor} - colstates::Vector{Tuple{Int64,Int64}} - rows::UnitRange{Int64} # rows to scan over - row::Int64 # current row -end - -""" -Create cursor to iterate over records. In parallel mode, multiple remote cursors can be created and iterated on in parallel. - -```julia -RecordCursor(par::Parquet.File; kwargs...) -``` - -Cursor options: -- `rows`: the row range to iterate through, all rows by default. -- `colnames`: the column names to retrieve; all by default -""" -function RecordCursor(par::Parquet.File; rows::UnitRange=1:nrows(par), colnames::Vector{Vector{String}}=colnames(par)) - colcursors = [ColCursor(par, colname; rows=rows, row=first(rows)) for colname in colnames] - sch = schema(par) - rectype = ntelemtype(sch, sch.schema[1]) - RecordCursor{rectype}(par, colnames, colcursors, Array{Tuple{Int64,Int64}}(undef, length(colcursors)), rows, first(rows)) -end - -eltype(::Type{RecordCursor{T}}) where {T} = T -length(cursor::RecordCursor) = length(cursor.rows) - -function Base.iterate(cursor::RecordCursor{T}, row) where {T} - (row > last(cursor.rows)) && (return nothing) - - states = cursor.colstates - cursors = cursor.colcursors - - colvals = Dict{Symbol,Any}() - col_repeat_state = Dict{Tuple{Int,Int},Int}() - for colid in 1:length(states) # for each column - colcursor = cursors[colid] - colstate = states[colid] - states[colid] = update_record(cursor.par, colvals, colid, colcursor, colstate, col_repeat_state) - end - cursor.row += 1 - _nt(colvals, T), cursor.row -end - -function Base.iterate(cursor::RecordCursor{T}) where {T} - cursor.row = first(cursor.rows) - cursor.colstates = [_start(colcursor) for colcursor in cursor.colcursors] - iterate(cursor, cursor.row) -end - -function _val_or_missing(dict::Dict{Symbol,Any}, k::Symbol, ::Type{T}) where {T} - v = get(dict, k, missing) - if isa(v, Vector) - elt = eltype(v) - if Dict{Symbol,Any} <: elt - nonmissing_elt = nonmissingtype(eltype(T)) - v = [el === missing ? el : _nt(el,nonmissing_elt) for el in v] - end - end - (isa(v, Dict{Symbol,Any}) ? _nt(v, nonmissingtype(T)) : v)::T -end - -@generated function _nt(dict::Dict{Symbol,Any}, ::Type{T}) where {T} - names = fieldnames(T) - strnames = ["$n" for n in names] - quote - return T(($([:(_val_or_missing(dict,Symbol($(strnames[i])),$(fieldtype(T,i)))) for i in 1:length(names)]...),)) - end -end - -default_init(::Type{Vector{T}}) where {T} = Vector{T}() -default_init(::Type{Dict{Symbol,Any}}) = Dict{Symbol,Any}() -default_init(::Type{T}) where {T} = ccall(:jl_new_struct_uninit, Any, (Any,), T)::T - -function update_record(par::Parquet.File, row::Dict{Symbol,Any}, colid::Int, colcursor::ColCursor, colcursor_state::Tuple{Int64,Int64}, col_repeat_state::Dict{Tuple{Int,Int},Int}) - colpos = colcursor.row - # iterate all repeated values from the column cursor (until it advances to the next row) - while !_done(colcursor, colcursor_state) - colval, colcursor_state = _next(colcursor, colcursor_state) # for each value, defn level, repn level in column - update_record(par, row, colid, colcursor, colval.value, colval.defn_level, colval.repn_level, col_repeat_state) # update record - (colcursor.row > colpos) && break - end - colcursor_state # return new colcursor state -end - -function update_record(par::Parquet.File, row::Dict{Symbol,Any}, colid::Int, colcursor::ColCursor, val, defn_level::Int64, repn_level::Int64, col_repeat_state::Dict{Tuple{Int,Int},Int}) - nameparts = colcursor.colname - symnameparts = colcursor.colnamesym - required_at = colcursor.required_at - repeated_at = colcursor.repeated_at - - lparts = length(nameparts) - sch = par.schema - F = row # the current field corresponding to the level in nameparts - Fdefn = 0 - Frepn = 0 - - # for each name part of colname (a field) - for idx in 1:lparts - colname = view(nameparts, 1:idx) - #@debug("updating part $colname of $nameparts isnull:$(val === nothing), def:$(defn_level), rep:$(repn_level)") - leaf = nameparts[idx] - symleaf = symnameparts[idx] - - required = required_at[idx] # determine whether field is optional and repeated - repeated = repeated_at[idx] - required || (Fdefn += 1) # if field is optional, increment defn level - repeated && (Frepn += 1) # if field can repeat, increment repn level - - defined = ((val === nothing) || (idx < lparts)) ? haskey(F, symleaf) : false - mustdefine = defn_level >= Fdefn - mustrepeat = repeated && (repn_level <= Frepn) - repidx = 0 - if mustrepeat - repkey = (colid, idx) - repidx = get(col_repeat_state, repkey, 0) - repidx += 1 - col_repeat_state[repkey] = repidx - end - nreps = (defined && isa(F[symleaf], Vector)) ? length(F[symleaf]) : 0 - - #@debug("repeat:$mustrepeat, nreps:$nreps, repidx:$repidx, defined:$defined, mustdefine:$mustdefine") - if mustrepeat && (nreps < repidx) - if !defined && mustdefine - Vtyp = elemtype(sch, colname) - Vrep = F[symleaf] = default_init(Vtyp) - else - Vrep = F[symleaf] - end - if length(Vrep) < repidx - resize!(Vrep, repidx) - if !isbits(eltype(Vrep)) - Vrep[repidx] = default_init(eltype(Vrep)) - end - end - F = Vrep[repidx] - elseif !defined && mustdefine - if idx == length(nameparts) - V = val - else - Vtyp = elemtype(sch, colname) - V = default_init(Vtyp) - end - F[symleaf] = V - F = V - else - F = get(F, symleaf, nothing) - end - end - nothing -end diff --git a/src/dataset.jl b/src/dataset.jl deleted file mode 100644 index 68fbe39..0000000 --- a/src/dataset.jl +++ /dev/null @@ -1,214 +0,0 @@ -const DATASET_METADATA_FILES = ("_common_metadata", "_metadata") - -""" - Parquet.Dataset(path; kwargs...) - -Returns the table contained in the parquet dataset in an Tables.jl compatible format. -A dataset comprises of multiple parquet files and optionally some metadata files. - -These options if provided are passed along while reading each parquet file in the dataset: -- `filter`: Filter function that takes the path to partitioned file and returns boolean to indicate whether to include the partition while loading. All partitions are loaded by default. -- `batchsize`: Maximum number of rows to read in each batch (default: row count of first row group). Applied to each file in the partition. -- `use_threads`: Whether to use threads while reading the file; applicable only for Julia v1.3 and later and switched on by default if julia processes is started with multiple threads. -- `column_generator`: Function to generate a partitioned column when not found in the partitioned table. Parameters provided to the function: table, column index, length of column to generate. Default implementation determines column values from the table path. - -One can easily convert the returned object to any Tables.jl compatible table e.g. DataFrames.DataFrame via - -``` -using DataFrames -df = DataFrame(read_parquet(path)) -``` -""" -struct Dataset <: Tables.AbstractColumns - path::String - filter::Function - ncols::Int - kwargs::NamedTuple{(:batchsize, :use_threads, :column_generator), Tuple{Union{Nothing,Signed}, Bool, Function}} - schema::Tables.Schema - lookup::Dict{Symbol, Int} # map column name => index - columns::Vector{AbstractVector} - tables::Vector{Table} - - function Dataset(path; - filter::Function=(path)->true, - batchsize::Union{Nothing,Signed}=nothing, - column_generator::Function=column_generator, - use_threads::Bool=(nthreads() > 1)) - - isdir(path) || error("Invalid Dataset path. Not a directory - $path") - sch = dataset_schema(string(path)) - ncols = length(sch.names) - lookup = Dict{Symbol, Int}(nm => i for (i, nm) in enumerate(sch.names)) - kwargs = (batchsize=batchsize, use_threads=use_threads, column_generator=dataset_column_generator) - new(path, filter, ncols, kwargs, sch, lookup, AbstractVector[], Table[]) - end -end - -const PARTITION_DATE_FORMATS = [dateformat"Y-m-d", dateformat"Y-m-d HH:MI:SS", dateformat"Y-m-dTHH:MI:SS"] -const PARTITION_DATETIME_FORMATS = [dateformat"Y-m-d HH:MI:SS", dateformat"Y-m-dTHH:MI:SS", dateformat"Y-m-d"] -function parse_date(missingstrval) - for format in PARTITION_DATE_FORMATS - try - return Date(missingstrval, format) - catch ex - (format == last(PARTITION_DATE_FORMATS)) && rethrow() - end - end -end -function parse_datetime(missingstrval) - for format in PARTITION_DATETIME_FORMATS - try - return DateTime(missingstrval, format) - catch ex - (format == last(PARTITION_DATETIME_FORMATS)) && rethrow() - end - end -end - -function dataset_column_generator(table::Table, colidx::Int, len::Int) - table_path = getfield(table, :path) - schema = getfield(table, :schema) - colname = schema.names[colidx] - coltype = schema.types[colidx] - - pattern = Regex("\\S+[\\/\\\\]?$(colname)=([a-zA-Z0-9 :\\-\\.]*)[\\/\\\\]?\\S+") - matches = match(pattern, table_path) - if (matches !== nothing) && (length(matches.captures) == 1) - missingstrval = matches.captures[1] - nm_coltype = nonmissingtype(coltype) - if nm_coltype <: Real - missingval = parse(nm_coltype, lowercase(missingstrval)) - elseif nm_coltype <: Date - missingval = parse_date(missingstrval) - elseif nm_coltype <: DateTime - missingval = parse_datetime(missingstrval) - elseif nm_coltype <: String - missingval = string(missingstrval) - else - error("Unhandled dataset partitioned column type $nm_coltype for column $colname of table $table_path") - end - else - missingval = nonmissingtype(coltype) === coltype ? undef : missing - end - - if missingval === undef || missingval === missing - Array{coltype}(missingval, len) - else - fill!(Array{coltype}(undef, len), missingval) - end -end - -function close(dataset::Dataset) - empty!(getfield(dataset, :columns)) - for table in getfield(dataset, :tables) - close(table) - end - empty!(getfield(dataset, :tables)) - nothing -end - -function dataset_schema(path::String) - schema = nothing - - # look for _common_metadata or _metadata file - for name in DATASET_METADATA_FILES - meta_file = joinpath(path, name) - if isfile(meta_file) - schema = tables_schema(Parquet.File(meta_file)) - break - end - end - - if schema === nothing - # else extract schema from any file in the dataset - for (root, dirs, files) in walkdir(path) - for file in files - full_filename = joinpath(root, file) - if is_par_file(full_filename) - schema = tables_schema(Parquet.File(full_filename)) - break - end - end - (schema === nothing) || break - end - end - - schema -end - -""" -Iterator to iterate over partitions of a parquet dataset, returned by the `Tables.partitions(dataset)` method. -Each partition is a Parquet.Table. -""" -struct DatasetPartitions - dataset::Dataset - ncols::Int - filter::Function - - function DatasetPartitions(dataset::Dataset, filter::Function) - new(dataset, getfield(dataset, :ncols), filter) - end -end - -function iterated_partition(partitions::DatasetPartitions, cursor) - partition = nothing - walker, root, files, fileidx, step = cursor - schema = getfield(partitions.dataset, :schema) - - while partition === nothing - if 0 < fileidx <= length(files) # we are iterating on files in a directory - file = files[fileidx] - fileidx += 1 - if !(file in DATASET_METADATA_FILES) - full_filename = joinpath(root, file) - if partitions.filter(full_filename) && is_par_file(full_filename) - partition = Table(full_filename, schema; getfield(partitions.dataset, :kwargs)...) - end - end - else # walk further into directory tree - itervals = (fileidx == 0) ? iterate(walker) : iterate(walker, step) - (itervals === nothing) && (return nothing) # end of directory tree - - dirinfo, step = itervals - root, dirs, files = dirinfo - fileidx = 1 - end - end - partition, (walker, root, files, fileidx, step) -end -Base.iterate(partitions::DatasetPartitions, cursor) = iterated_partition(partitions, cursor) -Base.iterate(partitions::DatasetPartitions) = iterated_partition(partitions, (walkdir(getfield(partitions.dataset, :path)), "", [], 0, nothing)) - -loaded(dataset::Dataset) = !isempty(getfield(dataset, :columns)) -function load(dataset::Dataset) - tables = getfield(dataset, :tables) - columns = getfield(dataset, :columns) - ncols = getfield(dataset, :ncols) - empty!(tables) - empty!(columns) - for table in Tables.partitions(dataset) - push!(tables, table) - if isempty(columns) - for colidx in 1:ncols - push!(columns, ChainedVector([Tables.getcolumn(table, colidx)])) - end - else - for colidx in 1:ncols - append!(columns[colidx], Tables.getcolumn(table, colidx)) - end - end - end - nothing -end - -Tables.istable(::Dataset) = true -Tables.columnaccess(::Dataset) = true -Tables.schema(d::Dataset) = getfield(d, :schema) -Tables.columnnames(d::Dataset) = getfield(d, :schema).names -Tables.columns(d::Dataset) = Tables.CopiedColumns(d) -Tables.getcolumn(d::Dataset, nm::Symbol) = Tables.getcolumn(d, getfield(d, :lookup)[nm]) -function Tables.getcolumn(d::Dataset, i::Int) - loaded(d) || load(d) - getfield(d, :columns)[i] -end -Tables.partitions(d::Dataset) = DatasetPartitions(d, getfield(d, :filter)) \ No newline at end of file diff --git a/src/delta.jl b/src/delta.jl new file mode 100644 index 0000000..b8ccab1 --- /dev/null +++ b/src/delta.jl @@ -0,0 +1,620 @@ +# DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, and DELTA_BYTE_ARRAY (Encodings.md, Parquet 2.13.0). +# Value reconstruction wraps in two's complement at the physical width, as the specification +# requires; only structural arithmetic (counts, sizes, positions) is checked. + +const DELTA_BLOCK_SIZE = 128 +const DELTA_MINIBLOCKS = 4 +const DELTA_MINIBLOCK_VALUES = DELTA_BLOCK_SIZE ÷ DELTA_MINIBLOCKS + +struct DeltaHeader + blocksize::Int + miniblocks::Int + miniblockvalues::Int + count::Int + first::Int64 +end + +function _readuleb128(bytes::AbstractVector{UInt8}, offset::Int) + value = UInt64(0) + position = offset + for index in 0:9 + _requirebytes(bytes, position, 1) + byte = bytes[position] + position += 1 + index == 9 && byte > 0x01 && throw(FormatError("delta varint overflows 64 bits")) + value |= UInt64(byte & 0x7f) << (7 * index) + iszero(byte & 0x80) && return value, position + end + throw(FormatError("delta varint is longer than 10 bytes")) +end + +function _readzigzag(bytes::AbstractVector{UInt8}, offset::Int) + raw, position = _readuleb128(bytes, offset) + return reinterpret(Int64, (raw >> 1) ⊻ (-(raw & 0x01))), position +end + +function _writeuleb128!(output::Vector{UInt8}, value::UInt64) + while value >= 0x80 + push!(output, UInt8(value & 0x7f) | 0x80) + value >>= 7 + end + push!(output, UInt8(value)) + return +end + +function _writezigzag!(output::Vector{UInt8}, value::Int32) + _writeuleb128!(output, UInt64(reinterpret(UInt32, (value << 1) ⊻ (value >> 31)))) + return +end + +function _writezigzag!(output::Vector{UInt8}, value::Int64) + _writeuleb128!(output, reinterpret(UInt64, (value << 1) ⊻ (value >> 63))) + return +end + +function _checkedposition(position::Int, count::Integer) + 0 <= count <= typemax(Int) - position || throw(FormatError("delta stream position overflows")) + return position + Int(count) +end + +function _structural(value::Integer) + 0 <= value <= typemax(Int) || throw(FormatError("delta structural size $value overflows")) + return Int(value) +end + +# Charge `count` values of `width` bytes to the page limit without overflowing the product. +function _checkbytes(count::Integer, width::Integer, limits::Limits) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + width > 0 || throw(ArgumentError("byte width must be positive")) + count <= limits.max_page_bytes ÷ width && return Int64(count * width) + requested = count <= typemax(Int64) ÷ width ? Int64(count * width) : typemax(Int64) + throw(LimitError(:page_bytes, requested, limits.max_page_bytes)) +end + +function _deltaint(raw::UInt64, what::String) + raw <= typemax(Int32) || throw(FormatError("delta $what $raw exceeds Int32")) + return Int(raw) +end + +function _checkdeltalayout(blocksize::Int, miniblocks::Int) + blocksize > 0 && blocksize % 128 == 0 || throw(FormatError("delta block size $blocksize is not a positive multiple of 128")) + 0 < miniblocks <= blocksize || throw(FormatError("delta miniblock count $miniblocks is invalid for block size $blocksize")) + blocksize % miniblocks == 0 || throw(FormatError("delta block size $blocksize is not divisible by $miniblocks miniblocks")) + miniblockvalues = blocksize ÷ miniblocks + miniblockvalues % 32 == 0 || throw(FormatError("delta miniblock holds $miniblockvalues values, not a multiple of 32")) + return miniblockvalues +end + +function _readdeltaheader(bytes::AbstractVector{UInt8}, offset::Int, count::Int, limits::Limits) + rawblocksize, position = _readuleb128(bytes, offset) + rawminiblocks, position = _readuleb128(bytes, position) + rawcount, position = _readuleb128(bytes, position) + first, position = _readzigzag(bytes, position) + blocksize = _deltaint(rawblocksize, "block size") + miniblocks = _deltaint(rawminiblocks, "miniblock count") + total = _deltaint(rawcount, "value count") + _checklimit(:container_elements, blocksize, limits.max_container_elements) + _checklimit(:container_elements, miniblocks, limits.max_container_elements) + _checklimit(:container_elements, total, limits.max_container_elements) + miniblockvalues = _checkdeltalayout(blocksize, miniblocks) + total == count || throw(FormatError("delta value count $total does not match the expected $count")) + return DeltaHeader(blocksize, miniblocks, miniblockvalues, total, first), position +end + +function _deltawidth(::Type{Int64}) + return 64 +end + +function _deltawidth(::Type{Int32}) + return 32 +end + +function _deltavalue(::Type{Int64}, accumulator::UInt64) + return reinterpret(Int64, accumulator) +end + +function _deltavalue(::Type{Int32}, accumulator::UInt64) + return reinterpret(Int32, accumulator % UInt32) +end + +function _deltasigned(::Type{Int64}, value::Int64, what::String) + return value +end + +function _deltasigned(::Type{Int32}, value::Int64, what::String) + typemin(Int32) <= value <= typemax(Int32) || throw(FormatError("delta $what $value exceeds Int32")) + return Int32(value) +end + +function _deltamask(width::Int) + width == 64 && return typemax(UInt64) + return (UInt64(1) << width) - UInt64(1) +end + +function _unpackdeltas!(::Type{B}, output::AbstractVector{T}, index::Int, count::Int, + bytes::AbstractVector{UInt8}, offset::Int, width::Int, accumulator::UInt64, + mindelta::UInt64) where {B<:Unsigned,T} + mask = B(_deltamask(width)) + buffer = zero(B) + bits = 0 + position = offset + @inbounds for slot in 0:(count - 1) + while bits < width + buffer |= B(bytes[position]) << bits + position += 1 + bits += 8 + end + accumulator += mindelta + UInt64(buffer & mask) + buffer >>= width + bits -= width + output[index + slot] = _deltavalue(T, accumulator) + end + return accumulator +end + +function _unpackminiblock!(output::AbstractVector{T}, index::Int, count::Int, + bytes::AbstractVector{UInt8}, offset::Int, width::Int, accumulator::UInt64, + mindelta::UInt64) where {T} + if width == 0 + @inbounds for slot in 0:(count - 1) + accumulator += mindelta + output[index + slot] = _deltavalue(T, accumulator) + end + return accumulator + end + width <= 56 && return _unpackdeltas!(UInt64, output, index, count, bytes, offset, width, accumulator, mindelta) + return _unpackdeltas!(UInt128, output, index, count, bytes, offset, width, accumulator, mindelta) +end + +# Width bytes of miniblocks that hold no values may be arbitrary; they are validated only when used. +function _readminiblockwidths!(widths::Vector{UInt8}, bytes::AbstractVector{UInt8}, offset::Int) + _requirebytes(bytes, offset, length(widths)) + @inbounds for index in eachindex(widths) + widths[index] = bytes[offset + index - 1] + end + return offset + length(widths) +end + +function _miniblockwidth(::Type{T}, width::UInt8) where {T} + width <= _deltawidth(T) || throw(FormatError("delta miniblock bit width $width exceeds the physical width $(_deltawidth(T))")) + return Int(width) +end + +function _miniblockpayload(header::DeltaHeader, width::Int, limits::Limits) + payload = (Int64(header.miniblockvalues) * Int64(width)) >> 3 + _checklimit(:page_bytes, payload, limits.max_page_bytes) + return _structural(payload) +end + +function _decodedeltablock!(output::AbstractVector{T}, index::Int, bytes::AbstractVector{UInt8}, + offset::Int, header::DeltaHeader, widths::Vector{UInt8}, accumulator::UInt64, + limits::Limits) where {T} + signedmin, position = _readzigzag(bytes, offset) + mindelta = reinterpret(UInt64, Int64(_deltasigned(T, signedmin, "minimum delta"))) + position = _readminiblockwidths!(widths, bytes, position) + count = length(output) + for miniblock in 1:header.miniblocks + index > count && break + width = _miniblockwidth(T, widths[miniblock]) + payload = _miniblockpayload(header, width, limits) + _checkedposition(position, payload) + _requirebytes(bytes, position, payload) + available = min(header.miniblockvalues, count - index + 1) + accumulator = _unpackminiblock!(output, index, available, bytes, position, width, accumulator, mindelta) + position += payload + index += available + end + return index, position, accumulator +end + +function decode_delta_binary_packed!(output::AbstractVector{T}, bytes::AbstractVector{UInt8}; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) where {T<:Union{Int32,Int64}} + count = length(output) + header, position = _readdeltaheader(bytes, Int(offset), count, limits) + count == 0 && return position + first = _deltasigned(T, header.first, "first value") + output[firstindex(output)] = first + count == 1 && return position + _requirebytes(bytes, position, header.miniblocks + 1) + widthscharge = _reservearray!(budget, UInt8, header.miniblocks) + widths = Vector{UInt8}(undef, header.miniblocks) + try + accumulator = reinterpret(UInt64, Int64(first)) + index = firstindex(output) + 1 + while index <= lastindex(output) + index, position, accumulator = _decodedeltablock!(output, index, + bytes, position, header, widths, accumulator, limits) + end + finally + _release!(budget, widthscharge) + end + return position +end + +function _decode_delta_binary_packed(::Type{T}, bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) where {T<:Union{Int32,Int64}} + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + _checkbytes(count, sizeof(T), limits) + outputcharge = _reservearray!(budget, T, count) + output = Vector{T}(undef, _structural(count)) + position = try + decode_delta_binary_packed!(output, bytes; offset=offset, limits=limits, + budget=budget) + catch + _release!(budget, outputcharge) + rethrow() + end + return output, position, outputcharge +end + +function decode_delta_binary_packed(::Type{T}, bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) where {T<:Union{Int32,Int64}} + output, position, _ = _decode_delta_binary_packed(T, bytes, count; + offset=offset, limits=limits, budget=budget) + return output, position +end + +function _bitwidth(value::UInt64) + return 64 - leading_zeros(value) +end + +function _packvalues!(output::Vector{UInt8}, values::AbstractVector{UInt64}, width::Int) + width == 0 && return + buffer = UInt128(0) + bits = 0 + for value in values + buffer |= UInt128(value) << bits + bits += width + while bits >= 8 + push!(output, UInt8(buffer & 0xff)) + buffer >>= 8 + bits -= 8 + end + end + bits > 0 && push!(output, UInt8(buffer & 0xff)) + return +end + +function _encodedeltablock!(output::Vector{UInt8}, deltas::Vector{T}, count::Int, + relative::Vector{UInt64}) where {T} + mindelta = deltas[1] + for index in 2:count + mindelta = min(mindelta, deltas[index]) + end + _writezigzag!(output, mindelta) + for index in 1:DELTA_BLOCK_SIZE + relative[index] = index <= count ? UInt64(reinterpret(unsigned(T), deltas[index] - mindelta)) : UInt64(0) + end + widths = zeros(UInt8, DELTA_MINIBLOCKS) + for miniblock in 1:DELTA_MINIBLOCKS + start = (miniblock - 1) * DELTA_MINIBLOCK_VALUES + 1 + start <= count || break + widths[miniblock] = UInt8(_bitwidth(maximum(@view relative[start:(start + DELTA_MINIBLOCK_VALUES - 1)]))) + end + append!(output, widths) + for miniblock in 1:DELTA_MINIBLOCKS + start = (miniblock - 1) * DELTA_MINIBLOCK_VALUES + 1 + start <= count || break + _packvalues!(output, @view(relative[start:(start + DELTA_MINIBLOCK_VALUES - 1)]), Int(widths[miniblock])) + end + return +end + +function encode_delta_binary_packed(values::AbstractVector{T}) where {T<:Union{Int32,Int64}} + count = length(values) + count <= typemax(Int32) || throw(ArgumentError("too many values for DELTA_BINARY_PACKED")) + output = UInt8[] + _writeuleb128!(output, UInt64(DELTA_BLOCK_SIZE)) + _writeuleb128!(output, UInt64(DELTA_MINIBLOCKS)) + _writeuleb128!(output, UInt64(count)) + _writezigzag!(output, count == 0 ? zero(T) : T(first(values))) + count <= 1 && return output + deltas = Vector{T}(undef, DELTA_BLOCK_SIZE) + relative = Vector{UInt64}(undef, DELTA_BLOCK_SIZE) + previous = T(first(values)) + index = firstindex(values) + 1 + while index <= lastindex(values) + blockcount = min(DELTA_BLOCK_SIZE, lastindex(values) - index + 1) + for slot in 1:blockcount + value = T(values[index + slot - 1]) + deltas[slot] = value - previous + previous = value + end + _encodedeltablock!(output, deltas, blockcount, relative) + index += blockcount + end + return output +end + +function _decode_delta_length_byte_array_offsets(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + _checkbytes(count, 4, limits) + size = _structural(count) + lengthscharge = _reservearray!(budget, Int32, size) + offsetscharge = try + _reservearray!(budget, Int, size + 1) + catch + _release!(budget, lengthscharge) + rethrow() + end + lengths = Vector{Int32}(undef, size) + offsets = Vector{Int}(undef, size + 1) + try + position = decode_delta_binary_packed!(lengths, bytes; offset=offset, + limits=limits, budget=budget) + offsets[1] = position + total = Int64(0) + @inbounds for index in 1:size + length = lengths[index] + length >= 0 || throw(FormatError( + "negative DELTA_LENGTH_BYTE_ARRAY length")) + _checklimit(:string_bytes, length, limits.max_string_bytes) + total += length + _checklimit(:page_bytes, total, limits.max_page_bytes) + offsets[index + 1] = _checkedposition(position, total) + end + last = _checkedposition(position, total) + _requirebytes(bytes, position, last - position) + _release!(budget, lengthscharge) + return offsets, last, offsetscharge + catch + _release!(budget, _materializedsum(lengthscharge, offsetscharge)) + rethrow() + end +end + +function decode_delta_length_byte_array_offsets(bytes::AbstractVector{UInt8}, + count::Integer; offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + offsets, position, _ = _decode_delta_length_byte_array_offsets(bytes, count; + offset=offset, limits=limits, budget=budget) + return offsets, position +end + +function _collectbytearrays(bytes::AbstractVector{UInt8}, offsets::Vector{Int}, + budget::_LiveByteBudget) + count = length(offsets) - 1 + charge = _materializedarraybytes(Vector{UInt8}, count) + for index in 1:count + length = offsets[index + 1] - offsets[index] + length >= 0 || throw(FormatError("byte-array offsets are not monotonic")) + charge = _materializedsum(charge, + _materializedarraybytes(UInt8, length)) + end + _reserve!(budget, charge) + output = Vector{Vector{UInt8}}(undef, count) + try + for index in eachindex(output) + first = offsets[index] + length = offsets[index + 1] - first + value = Vector{UInt8}(undef, length) + length == 0 || copyto!(value, 1, bytes, first, length) + output[index] = value + end + catch + _release!(budget, charge) + rethrow() + end + return output, charge +end + +function _decode_delta_length_byte_array(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + offsets, position, offsetscharge = _decode_delta_length_byte_array_offsets( + bytes, count; offset=offset, limits=limits, budget=budget) + output, outputcharge = try + _collectbytearrays(bytes, offsets, budget) + catch + _release!(budget, offsetscharge) + rethrow() + end + _release!(budget, offsetscharge) + return output, position, outputcharge +end + +function decode_delta_length_byte_array(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + output, position, _ = _decode_delta_length_byte_array(bytes, count; + offset=offset, limits=limits, budget=budget) + return output, position +end + +function _bytearraylengths(values) + lengths = Vector{Int32}(undef, length(values)) + for (index, value) in enumerate(values) + bytes = value isa AbstractString ? codeunits(value) : value + length(bytes) <= typemax(Int32) || throw(ArgumentError("byte array exceeds Int32 length")) + lengths[index] = Int32(length(bytes)) + end + return lengths +end + +function encode_delta_length_byte_array(values) + output = encode_delta_binary_packed(_bytearraylengths(values)) + for value in values + append!(output, value isa AbstractString ? codeunits(value) : value) + end + return output +end + +function _deltabytearraylayout(prefixes::Vector{Int32}, + suffixoffsets::Vector{Int}, limits::Limits, budget::_LiveByteBudget) + count = length(prefixes) + offsetscharge = _reservearray!(budget, Int, count + 1) + offsets = Vector{Int}(undef, count + 1) + try + offsets[1] = 1 + previous = Int64(0) + total = Int64(0) + @inbounds for index in 1:count + prefix = Int64(prefixes[index]) + 0 <= prefix <= previous || throw(FormatError( + "DELTA_BYTE_ARRAY prefix length $prefix exceeds the previous value")) + length = prefix + (suffixoffsets[index + 1] - suffixoffsets[index]) + _checklimit(:string_bytes, length, limits.max_string_bytes) + total += length + _checklimit(:page_bytes, total, limits.max_page_bytes) + offsets[index + 1] = _checkedposition(offsets[index], length) + previous = length + end + return offsets, _structural(total), offsetscharge + catch + _release!(budget, offsetscharge) + rethrow() + end +end + +function _fillbytearrays!(data::Vector{UInt8}, offsets::Vector{Int}, prefixes::Vector{Int32}, + bytes::AbstractVector{UInt8}, suffixoffsets::Vector{Int}) + @inbounds for index in eachindex(prefixes) + prefix = Int(prefixes[index]) + target = offsets[index] + prefix == 0 || copyto!(data, target, data, offsets[index - 1], prefix) + suffixlength = suffixoffsets[index + 1] - suffixoffsets[index] + suffixlength == 0 || copyto!(data, target + prefix, bytes, suffixoffsets[index], suffixlength) + end + return +end + +function _decode_delta_byte_array_buffer(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + _checkbytes(count, 4, limits) + size = _structural(count) + prefixescharge = _reservearray!(budget, Int32, size) + prefixes = Vector{Int32}(undef, size) + suffixcharge = Int64(0) + offsetscharge = Int64(0) + datacharge = Int64(0) + try + position = decode_delta_binary_packed!(prefixes, bytes; offset=offset, + limits=limits, budget=budget) + suffixoffsets, position, suffixcharge = + _decode_delta_length_byte_array_offsets(bytes, count; offset=position, + limits=limits, budget=budget) + offsets, total, offsetscharge = _deltabytearraylayout(prefixes, + suffixoffsets, limits, budget) + datacharge = _reservearray!(budget, UInt8, total) + data = Vector{UInt8}(undef, total) + _fillbytearrays!(data, offsets, prefixes, bytes, suffixoffsets) + _release!(budget, _materializedsum(prefixescharge, suffixcharge)) + retained = _materializedsum(datacharge, offsetscharge) + return data, offsets, position, retained, offsetscharge + catch + charge = _materializedsum(prefixescharge, suffixcharge) + charge = _materializedsum(charge, offsetscharge) + charge = _materializedsum(charge, datacharge) + _release!(budget, charge) + rethrow() + end +end + +function decode_delta_byte_array_buffer(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + data, offsets, position, _, _ = _decode_delta_byte_array_buffer(bytes, + count; offset=offset, limits=limits, budget=budget) + return data, offsets, position +end + +function _decode_delta_byte_array(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + data, offsets, position, retained, _ = _decode_delta_byte_array_buffer( + bytes, count; offset=offset, limits=limits, budget=budget) + output, outputcharge = try + _collectbytearrays(data, offsets, budget) + catch + _release!(budget, retained) + rethrow() + end + _release!(budget, retained) + return output, position, outputcharge +end + +function decode_delta_byte_array(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + output, position, _ = _decode_delta_byte_array(bytes, count; + offset=offset, limits=limits, budget=budget) + return output, position +end + +function _decode_delta_byte_array_fixed(bytes::AbstractVector{UInt8}, count::Integer, + width::Integer; + offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + width > 0 || throw(FormatError("fixed byte-array width must be positive")) + _checklimit(:string_bytes, width, limits.max_string_bytes) + data, offsets, position, retained, offsetscharge = + _decode_delta_byte_array_buffer(bytes, count; offset=offset, + limits=limits, budget=budget) + size = _structural(count) + fixedwidth = _structural(width) + objectcharge = Int64(0) + try + @inbounds for index in 1:size + offsets[index + 1] - offsets[index] == width || throw(FormatError( + "DELTA_BYTE_ARRAY value length differs from the fixed width $width")) + end + objectcharge = _reserveobjects!(budget) + output = reshape(data, fixedwidth, size) + _release!(budget, offsetscharge) + outputcharge = _materializedsum(retained - offsetscharge, objectcharge) + return output, position, outputcharge + catch + _release!(budget, _materializedsum(retained, objectcharge)) + rethrow() + end +end + +function decode_delta_byte_array_fixed(bytes::AbstractVector{UInt8}, count::Integer, + width::Integer; offset::Integer=1, limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + output, position, _ = _decode_delta_byte_array_fixed(bytes, count, width; + offset=offset, limits=limits, budget=budget) + return output, position +end + +function _commonprefix(previous::AbstractVector{UInt8}, current::AbstractVector{UInt8}) + limit = min(length(previous), length(current)) + index = 0 + @inbounds while index < limit && previous[index + 1] == current[index + 1] + index += 1 + end + return index +end + +function encode_delta_byte_array(values) + count = length(values) + prefixes = Vector{Int32}(undef, count) + suffixes = Vector{SubArray{UInt8,1,Vector{UInt8},Tuple{UnitRange{Int}},true}}(undef, count) + previous = UInt8[] + for (index, value) in enumerate(values) + current = Vector{UInt8}(value isa AbstractString ? codeunits(value) : value) + length(current) <= typemax(Int32) || throw(ArgumentError("byte array exceeds Int32 length")) + prefix = _commonprefix(previous, current) + prefixes[index] = Int32(prefix) + suffixes[index] = @view current[(prefix + 1):end] + previous = current + end + output = encode_delta_binary_packed(prefixes) + append!(output, encode_delta_length_byte_array(suffixes)) + return output +end + +function encode_delta_byte_array_fixed(values::AbstractMatrix{UInt8}) + size(values, 1) > 0 || throw(ArgumentError("fixed byte-array width must be positive")) + return encode_delta_byte_array(eachcol(values)) +end diff --git a/src/dictionary.jl b/src/dictionary.jl new file mode 100644 index 0000000..30ff97c --- /dev/null +++ b/src/dictionary.jl @@ -0,0 +1,121 @@ +struct DecodedDictionary{T} + values::Vector{T} +end + +function _isdictionaryencoding(encoding::Metadata.Encoding.T) + return encoding == Metadata.Encoding.PLAIN_DICTIONARY || + encoding == Metadata.Encoding.RLE_DICTIONARY +end + +function _checkdictionarypageencoding(encoding::Metadata.Encoding.T) + encoding == Metadata.Encoding.PLAIN && return + encoding == Metadata.Encoding.PLAIN_DICTIONARY && return + throw(FormatError("dictionary page encoding $encoding is not PLAIN")) +end + +function _decodedictionarypage(::Type{T}, frame, md::Metadata.ColumnMetaData, + node::SchemaNode, limits::Limits, budget::_LiveByteBudget) where {T} + header = frame.header.dictionary_page_header + count = Int(header.num_values) + count >= 0 || throw(FormatError("negative dictionary entry count")) + _checklimit(:container_elements, count, limits.max_container_elements) + _checkdictionarypageencoding(header.encoding) + bytes = decompresspage(frame, md.codec; limits=limits, budget=budget) + values, position = _decodevalues(T, bytes, count, _fixedwidth(node), 1, limits) + position == length(bytes) + 1 || + throw(FormatError("dictionary page has $(length(bytes) - position + 1) trailing bytes")) + return DecodedDictionary(values) +end + +function _decodedictionaryindices(bytes::AbstractVector{UInt8}, count::Int, offset::Int, + limits::Limits) + count >= 0 || throw(ArgumentError("dictionary index count must be nonnegative")) + if count == 0 && offset == length(bytes) + 1 + return UInt64[], offset + end + _requirebytes(bytes, offset, 1) + bitwidth = Int(bytes[offset]) + bitwidth <= 32 || throw(FormatError("dictionary index bit width $bitwidth exceeds 32")) + return decode_hybrid(bytes, count, bitwidth; offset=offset + 1, limits=limits) +end + +function _dictionarycopy(value::Vector{UInt8}) + return copy(value) +end + +function _dictionarycopy(value) + return value +end + +function _lookupdictionary(dictionary::DecodedDictionary{T}, indices::Vector{UInt64}) where {T} + output = Vector{T}(undef, length(indices)) + count = UInt64(length(dictionary.values)) + for index in eachindex(indices) + raw = indices[index] + raw < count || throw(FormatError("dictionary index $raw is outside a dictionary of $count entries")) + output[index] = _dictionarycopy(dictionary.values[Int(raw) + 1]) + end + return output +end + +function _dictionarykey(value::Float32) + return reinterpret(UInt32, value) +end + +function _dictionarykey(value::Float64) + return reinterpret(UInt64, value) +end + +function _dictionarykey(value::AbstractString) + return String(value) +end + +function _dictionarykey(value::AbstractVector{UInt8}) + return String(collect(value)) +end + +function _dictionarykey(value) + return value +end + +function _dictionaryentries(column) + T = Base.nonmissingtype(eltype(column.values)) + values = T[] + indices = UInt64[] + sizehint!(indices, length(column.values)) + lookup = Dict{Any,UInt64}() + for rawvalue in column.values + ismissing(rawvalue) && continue + value = convert(T, rawvalue) + key = _dictionarykey(value) + index = get(lookup, key, nothing) + if index === nothing + index = UInt64(length(values)) + push!(values, value) + lookup[key] = index + end + push!(indices, index) + end + return values, indices +end + +function _dictionarybitwidth(count::Int) + count >= 0 || throw(ArgumentError("dictionary entry count must be nonnegative")) + count <= 1 && return 0 + return 64 - leading_zeros(UInt64(count - 1)) +end + +function _encodedictionaryindices(indices::Vector{UInt64}, bitwidth::Int) + output = UInt8[UInt8(bitwidth)] + isempty(indices) && return output + if all(==(first(indices)), indices) + _writehybridvarint!(output, UInt64(length(indices)) << 1) + value = first(indices) + for index in 0:(cld(bitwidth, 8) - 1) + push!(output, UInt8((value >> (8 * index)) & 0xff)) + end + return output + end + append!(output, encode_hybrid(indices, bitwidth)) + return output +end diff --git a/src/dremel.jl b/src/dremel.jl new file mode 100644 index 0000000..e338672 --- /dev/null +++ b/src/dremel.jl @@ -0,0 +1,64 @@ +# Physical leaf streams use one repetition and definition level per entry, while +# storing only values whose definition reaches the leaf maximum. + +function _validateleafstream(repetition::Vector{UInt64}, definition::Vector{UInt64}, + values::Vector, maxrepetition::Integer, maxdefinition::Integer, expectedrows) + maxrepetition >= 0 || throw(FormatError("negative maximum repetition level")) + maxdefinition >= maxrepetition || + throw(FormatError("maximum repetition level exceeds maximum definition level")) + length(repetition) == length(definition) || + throw(FormatError("leaf stream repetition and definition counts differ")) + rows = 0 + present = 0 + for index in eachindex(repetition, definition) + repetitionlevel = repetition[index] + definitionlevel = definition[index] + repetitionlevel <= maxrepetition || + throw(FormatError("repetition level $repetitionlevel exceeds the maximum $maxrepetition")) + definitionlevel <= maxdefinition || + throw(FormatError("definition level $definitionlevel exceeds the maximum $maxdefinition")) + repetitionlevel <= definitionlevel || + throw(FormatError("repetition level $repetitionlevel exceeds definition level $definitionlevel")) + iszero(repetitionlevel) && (rows += 1) + definitionlevel == maxdefinition && (present += 1) + end + isempty(repetition) || iszero(first(repetition)) || + throw(FormatError("column chunk starts with repetition level $(first(repetition))")) + present == length(values) || + throw(FormatError("leaf stream has $(length(values)) dense values for $present present entries")) + if expectedrows !== nothing + expectedrows >= 0 || throw(FormatError("negative expected row count")) + rows == expectedrows || + throw(FormatError("leaf stream has $rows rows but $expectedrows were expected")) + end + return +end + +struct LeafStream{T} + repetition::Vector{UInt64} + definition::Vector{UInt64} + values::Vector{T} + + function LeafStream{T}(repetition::Vector{UInt64}, definition::Vector{UInt64}, + values::Vector{T}, maxrepetition::Integer, maxdefinition::Integer; + expected_rows=nothing) where {T} + _validateleafstream(repetition, definition, values, maxrepetition, maxdefinition, + expected_rows) + return new{T}(repetition, definition, values) + end +end + +function LeafStream(repetition::Vector{UInt64}, definition::Vector{UInt64}, + values::Vector{T}, maxrepetition::Integer, maxdefinition::Integer; + expected_rows=nothing) where {T} + return LeafStream{T}(repetition, definition, values, maxrepetition, maxdefinition; + expected_rows=expected_rows) +end + +function Base.length(stream::LeafStream) + return length(stream.repetition) +end + +function Base.isempty(stream::LeafStream) + return isempty(stream.repetition) +end diff --git a/src/errors.jl b/src/errors.jl new file mode 100644 index 0000000..2be2db3 --- /dev/null +++ b/src/errors.jl @@ -0,0 +1,343 @@ +struct FormatError <: Exception + message::String +end + +function Base.showerror(io::IO, err::FormatError) + print(io, "invalid Parquet file: ", err.message) + return +end + +struct UnsupportedFeatureError <: Exception + message::String +end + +function Base.showerror(io::IO, err::UnsupportedFeatureError) + print(io, "unsupported Parquet feature: ", err.message) + return +end + +struct LimitError <: Exception + resource::Symbol + requested::Int64 + maximum::Int64 +end + +function Base.showerror(io::IO, err::LimitError) + print(io, "Parquet ", err.resource, " limit exceeded: requested ", err.requested, + ", maximum ", err.maximum) + return +end + +Base.@kwdef struct Limits + max_footer_bytes::Int64 = 64 * 1024 * 1024 + max_page_header_bytes::Int64 = 16 * 1024 * 1024 + max_page_bytes::Int64 = 1024 * 1024 * 1024 + max_page_index_bytes::Int64 = 64 * 1024 * 1024 + max_statistics_value_bytes::Int64 = 4096 + max_materialized_bytes::Int64 = 2 * 1024 * 1024 * 1024 + max_schema_name_bytes::Int64 = 1024 * 1024 + max_string_bytes::Int64 = 256 * 1024 * 1024 + max_decimal_bytes::Int64 = 1024 * 1024 + max_container_elements::Int64 = 100_000_000 + max_metadata_depth::Int = 128 +end + +mutable struct _LiveByteBudget + maximum::Int64 + used::Int64 + lock::ReentrantLock +end + +const _MATERIALIZED_ARRAY_HEADER_BYTES = Int64(64) +const _MATERIALIZED_OBJECT_BYTES = Int64(128) + +function _LiveByteBudget(limits::Limits) + return _LiveByteBudget(limits.max_materialized_bytes, Int64(0), ReentrantLock()) +end + +function _budgetrequest(current::Int64, bytes::Integer, maximum::Int64, + resource::Symbol) + bytes >= 0 || throw(ArgumentError("cannot reserve a negative byte count")) + bytes <= typemax(Int64) || throw(LimitError(resource, typemax(Int64), maximum)) + requested = try + Base.checked_add(current, Int64(bytes)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(resource, typemax(Int64), maximum)) + end + requested <= maximum || throw(LimitError(resource, requested, maximum)) + return requested +end + +function _reserve!(budget::_LiveByteBudget, bytes::Integer; + resource::Symbol=:materialized_bytes) + lock(budget.lock) + try + budget.used = _budgetrequest(budget.used, bytes, budget.maximum, resource) + finally + unlock(budget.lock) + end + return +end + +function _release!(budget::_LiveByteBudget, bytes::Integer) + bytes >= 0 || throw(ArgumentError("cannot release a negative byte count")) + bytes <= typemax(Int64) || throw(ArgumentError("released byte count exceeds Int64")) + lock(budget.lock) + try + value = Int64(bytes) + value <= budget.used || throw(ArgumentError( + "cannot release $value bytes from a budget using $(budget.used) bytes")) + budget.used -= value + finally + unlock(budget.lock) + end + return +end + +function _budgetused(budget::_LiveByteBudget) + lock(budget.lock) + try + return budget.used + finally + unlock(budget.lock) + end +end + +function _materializedproduct(count::Integer, width::Integer) + count >= 0 || throw(ArgumentError("materialized element count must be nonnegative")) + width >= 0 || throw(ArgumentError("materialized element width must be nonnegative")) + count <= typemax(Int64) && width <= typemax(Int64) || + throw(LimitError(:materialized_bytes, typemax(Int64), typemax(Int64))) + return try + Base.checked_mul(Int64(count), Int64(width)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:materialized_bytes, typemax(Int64), typemax(Int64))) + end +end + +function _materializedsum(left::Integer, right::Integer) + left >= 0 && right >= 0 || + throw(ArgumentError("materialized byte counts must be nonnegative")) + left <= typemax(Int64) && right <= typemax(Int64) || + throw(LimitError(:materialized_bytes, typemax(Int64), typemax(Int64))) + return try + Base.checked_add(Int64(left), Int64(right)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:materialized_bytes, typemax(Int64), typemax(Int64))) + end +end + +function _materializedarraybytes(::Type{T}, count::Integer; + header::Bool=true) where {T} + width = Int64(Base.elsize(Vector{T})) + bytes = _materializedproduct(count, width) + (Base.isbitsunion(T) || Missing <: T) && + (bytes = _materializedsum(bytes, count)) + header && (bytes = _materializedsum(_MATERIALIZED_ARRAY_HEADER_BYTES, bytes)) + return bytes +end + +function _materializedbitbytes(count::Integer; header::Bool=true) + count >= 0 || throw(ArgumentError("materialized bit count must be nonnegative")) + payload = cld(Int128(count), Int128(8)) + payload <= typemax(Int64) || + throw(LimitError(:materialized_bytes, typemax(Int64), typemax(Int64))) + bytes = Int64(payload) + header && (bytes = _materializedsum(_MATERIALIZED_ARRAY_HEADER_BYTES, bytes)) + return bytes +end + +function _reservearray!(budget::_LiveByteBudget, ::Type{T}, count::Integer; + header::Bool=true) where {T} + bytes = _materializedarraybytes(T, count; header=header) + _reserve!(budget, bytes) + return bytes +end + +function _reservebits!(budget::_LiveByteBudget, count::Integer; header::Bool=true) + bytes = _materializedbitbytes(count; header=header) + _reserve!(budget, bytes) + return bytes +end + +function _reserveobjects!(budget::_LiveByteBudget, count::Integer=1) + bytes = _materializedproduct(count, _MATERIALIZED_OBJECT_BYTES) + _reserve!(budget, bytes) + return bytes +end + +struct _SchemaNameState + names::Set{String} + bytes::Int64 +end + +mutable struct _SchemaNameRegistry + state::_SchemaNameState + lock::ReentrantLock +end + +const _SCHEMA_NAME_REGISTRY = _SchemaNameRegistry( + _SchemaNameState(Set{String}(), Int64(0)), ReentrantLock()) + +function _schemanamecharge(name::String, maximum::Int64) + bytes = Int64(ncodeunits(name)) + payload = try + Base.checked_mul(bytes, Int64(2)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:schema_name_bytes, typemax(Int64), maximum)) + end + return _budgetrequest(Int64(256), payload, maximum, :schema_name_bytes) +end + +function _copyvalidatetablenames(names::AbstractVector{String}, + budget::_LiveByteBudget) + temporary = Int64(0) + try + temporary = _materializedsum(temporary, + _reservearray!(budget, String, length(names))) + validated = Vector{String}(undef, length(names)) + temporary = _materializedsum(temporary, + _reserveobjects!(budget)) + seen = Set{String}() + position = 1 + for index in eachindex(names) + name = names[index] + validated[position] = name + position += 1 + occursin('\0', name) && throw(UnsupportedFeatureError( + "Parquet.Table cannot represent a top-level field name containing NUL; " * + "use Parquet.File for low-level access")) + name in seen && throw(UnsupportedFeatureError( + "Parquet.Table requires unique top-level field names; " * + "use Parquet.File for low-level access")) + temporary = _materializedsum(temporary, + _reserveobjects!(budget)) + push!(seen, name) + end + return validated, temporary + catch + iszero(temporary) || _release!(budget, temporary) + rethrow() + end +end + +function _validatetablenames(names::AbstractVector{String}, + budget::_LiveByteBudget) + _, temporary = _copyvalidatetablenames(names, budget) + _release!(budget, temporary) + return +end + +function _validatetablenames(names::AbstractVector{String}) + return _validatetablenames(names, _LiveByteBudget(Limits())) +end + +function _internschemanames(names::AbstractVector{String}, limits::Limits, + budget::_LiveByteBudget) + validated, temporary = _copyvalidatetablenames(names, budget) + registry = _SCHEMA_NAME_REGISTRY + outputcharge = Int64(0) + output = nothing + additions = 0 + try + lock(registry.lock) + try + # max_schema_name_bytes bounds the new name bytes ONE operation may + # intern, not the process-global total: interned Symbols are immortal, + # so charging every operation against a shared lifetime cap would let a + # single hostile file exhaust it and deny every later file that carries + # any not-yet-interned name. The registry still tracks the global total + # as an observability metric. + state = registry.state + charge = Int64(0) + additions = 0 + for name in validated + name in state.names && continue + nextcharge = _schemanamecharge(name, + limits.max_schema_name_bytes) + charge = _budgetrequest(charge, nextcharge, + limits.max_schema_name_bytes, :schema_name_bytes) + additions += 1 + end + requested = _budgetrequest(state.bytes, charge, typemax(Int64), + :schema_name_bytes) + requested >= state.bytes || throw(AssertionError( + "schema-name registry byte accounting decreased")) + outputcharge = _reservearray!(budget, Symbol, + length(validated)) + output = Vector{Symbol}(undef, length(validated)) + symbols = something(output) + if additions > 0 + replacementcount = try + Base.checked_add(length(state.names), additions) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + typemax(Int64))) + end + replacementcharge = _materializedsum( + _materializedproduct(replacementcount, + _MATERIALIZED_OBJECT_BYTES), + _MATERIALIZED_OBJECT_BYTES) + _reserve!(budget, replacementcharge) + temporary = _materializedsum(temporary, + replacementcharge) + updated = Set{String}() + sizehint!(updated, replacementcount) + for name in state.names + push!(updated, name) + end + for name in validated + name in state.names || push!(updated, name) + end + for index in eachindex(validated) + symbols[index] = Symbol(validated[index]) + end + newstate = _SchemaNameState(updated, requested) + _release!(budget, temporary) + temporary = Int64(0) + registry.state = newstate + end + finally + unlock(registry.lock) + end + symbols = something(output) + if additions == 0 + for index in eachindex(validated) + symbols[index] = Symbol(validated[index]) + end + _release!(budget, temporary) + temporary = Int64(0) + end + return symbols + catch + iszero(outputcharge) || _release!(budget, outputcharge) + iszero(temporary) || _release!(budget, temporary) + rethrow() + end +end + +function _internschemanames(names::AbstractVector{String}, limits::Limits) + return _internschemanames(names, limits, _LiveByteBudget(limits)) +end + +function _internedschemanamebytes() + registry = _SCHEMA_NAME_REGISTRY + lock(registry.lock) + try + return registry.state.bytes + finally + unlock(registry.lock) + end +end + +function _checklimit(resource::Symbol, requested::Integer, maximum::Integer) + requested <= maximum && return + reported = requested > typemax(Int64) ? typemax(Int64) : + requested < typemin(Int64) ? typemin(Int64) : Int64(requested) + throw(LimitError(resource, reported, Int64(maximum))) +end diff --git a/src/footer.jl b/src/footer.jl new file mode 100644 index 0000000..92e946a --- /dev/null +++ b/src/footer.jl @@ -0,0 +1,78 @@ +const PARQUET_MAGIC = UInt8[0x50, 0x41, 0x52, 0x31] +const ENCRYPTED_MAGIC = UInt8[0x50, 0x41, 0x52, 0x45] + +struct Footer{B<:AbstractVector{UInt8}} + offset::Int64 + length::Int64 + encrypted::Bool + bytes::B +end + +mutable struct File{S<:AbstractSource,F<:Footer} + source::S + footer::F + @atomic closed::Bool +end + +function _readu32le(bytes::AbstractVector{UInt8}, offset::Int=1) + checkbounds(bytes, offset:(offset + 3)) + value = UInt32(bytes[offset]) | + UInt32(bytes[offset + 1]) << 8 | + UInt32(bytes[offset + 2]) << 16 | + UInt32(bytes[offset + 3]) << 24 + return value +end + +function _magic(bytes::AbstractVector{UInt8}, offset::Int=1) + checkbounds(bytes, offset:(offset + 3)) + first = bytes[offset] + second = bytes[offset + 1] + third = bytes[offset + 2] + fourth = bytes[offset + 3] + first == 0x50 && second == 0x41 && third == 0x52 && fourth == 0x31 && + return :plain + first == 0x50 && second == 0x41 && third == 0x52 && fourth == 0x45 && + return :encrypted + return :invalid +end + +function readfooter(src::AbstractSource, limits::Limits=Limits()) + total = _checkedsourcelength(src) + total >= 12 || throw(FormatError("file is shorter than the minimum 12 bytes")) + leading = _readrangeexact(src, total, Int64(0), Int64(4)) + leadingmagic = _magic(leading) + leadingmagic === :invalid && throw(FormatError("missing leading PAR1 or PARE magic")) + trailer = _readrangeexact(src, total, total - 8, Int64(8)) + trailingmagic = _magic(trailer, 5) + trailingmagic === :invalid && throw(FormatError("missing trailing PAR1 or PARE magic")) + leadingmagic === trailingmagic || throw(FormatError("leading and trailing magic differ")) + footerlength = Int64(_readu32le(trailer)) + footerlength <= total - 12 || throw(FormatError("footer length exceeds the file size")) + _checklimit(:footer_bytes, footerlength, limits.max_footer_bytes) + footeroffset = total - 8 - footerlength + bytes = _readrangeexact(src, total, footeroffset, footerlength) + return Footer(footeroffset, footerlength, leadingmagic === :encrypted, bytes) +end + +function File(input; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + src = source(input; budget=budget) + footer = try + readfooter(src, limits) + catch + input isa AbstractSource || close!(src) + rethrow() + end + return File(src, footer, false) +end + +function close!(file::File) + (@atomicswap file.closed = true) && return + close!(file.source) + return +end + +function Base.close(file::File) + close!(file) + return +end diff --git a/src/logical.jl b/src/logical.jl new file mode 100644 index 0000000..b0ed5c9 --- /dev/null +++ b/src/logical.jl @@ -0,0 +1,207 @@ +import Dates + +const _PARQUET_DATE_EPOCH = Dates.value(Dates.Date(1970, 1, 1)) + +function _requirelogicalphysical(element::Metadata.SchemaElement, + expected::Metadata.Type.T, annotation::Symbol) + element.type_ == expected && return + throw(FormatError("$annotation annotation on $(repr(element.name)) requires " * + "physical type $expected, got $(element.type_)")) +end + +function _logicalkind(element::Metadata.SchemaElement) + logical = element.logicalType + if logical !== nothing + if logical.STRING !== nothing + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :STRING) + return :string + elseif logical.DATE !== nothing + _requirelogicalphysical(element, Metadata.Type.INT32, :DATE) + return :date + end + kind = _temporallogicalkind(element) + kind === nothing || return kind + kind = _binarylogicalkind(element) + kind === nothing || return kind + kind = _decimallogicalkind(element) + kind === nothing || return kind + return nothing + end + converted = element.converted_type + if converted == Metadata.ConvertedType.UTF8 + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :UTF8) + return :string + elseif converted == Metadata.ConvertedType.DATE + _requirelogicalphysical(element, Metadata.Type.INT32, :DATE) + return :date + end + kind = _temporallogicalkind(element) + kind === nothing || return kind + kind = _binarylogicalkind(element) + kind === nothing || return kind + kind = _decimallogicalkind(element) + kind === nothing || return kind + return nothing +end + +function _logicalkind(node::SchemaNode) + return _logicalkind(node.element) +end + +function _logicaleltype(element::Metadata.SchemaElement, physical::Type) + kind = _logicalkind(element) + kind === nothing && return physical + kind === :string && return String + kind === :date && return Dates.Date + kind isa Union{_TimeLogicalKind,_TimestampLogicalKind,_IntegerLogicalKind} && + return _temporaljuliatype(kind) + logical = _binarylogicaleltype(kind, physical) + logical === nothing || return logical + logical = _decimallogicaleltype(kind) + logical === nothing || return logical + return physical +end + +function _logicaleltype(node::SchemaNode, physical::Type) + return _logicaleltype(node.element, physical) +end + +function _fromparquetdate(value::Int32) + ordinal = try + Base.checked_add(_PARQUET_DATE_EPOCH, Int64(value)) + catch err + err isa OverflowError || rethrow() + throw(FormatError("DATE value $value overflows the Julia date range")) + end + return Dates.Date(Dates.UTD(ordinal)) +end + +function _toparquetdate(value::Dates.Date) + days = try + Base.checked_sub(Dates.value(value), _PARQUET_DATE_EPOCH) + catch err + err isa OverflowError || rethrow() + throw(ArgumentError("DATE value $(repr(value)) is outside the Parquet INT32 day range")) + end + typemin(Int32) <= days <= typemax(Int32) || + throw(ArgumentError("DATE value $(repr(value)) is outside the Parquet INT32 day range")) + return Int32(days) +end + +function _fromparquetstring(value::AbstractVector{UInt8}, limits::Limits) + _checklimit(:string_bytes, length(value), limits.max_string_bytes) + isvalid(String, value) || throw(FormatError("STRING value contains invalid UTF-8")) + return String(copy(value)) +end + +function _toparquetstring(value::AbstractString, limits::Limits) + bytes = codeunits(value) + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + isvalid(String, bytes) || throw(ArgumentError("STRING value contains invalid UTF-8")) + return collect(bytes) +end + +function _logicalvalue(kind, element::Metadata.SchemaElement, value, limits::Limits) + ismissing(value) && return missing + if kind isa Union{_TimeLogicalKind,_TimestampLogicalKind,_IntegerLogicalKind} + return _temporallogicalvalue(kind, element, value) + elseif kind === :string + value isa AbstractVector{UInt8} || + throw(FormatError("STRING column $(repr(element.name)) contains a " * + "non-byte-array value")) + return _fromparquetstring(value, limits) + elseif kind === :date + value isa Int32 || + throw(FormatError("DATE column $(repr(element.name)) contains a non-INT32 value")) + return _fromparquetdate(value) + elseif kind === :decimal + return _decimallogicalvalue(kind, element, value, limits) + end + converted = _binarylogicalvalue(kind, element, value, limits) + converted === nothing && throw(FormatError( + "unsupported logical annotation $kind on $(repr(element.name))")) + return converted +end + +function _logicalvalue(element::Metadata.SchemaElement, value; limits::Limits=Limits()) + kind = _logicalkind(element) + kind === nothing && return value + return _logicalvalue(kind, element, value, limits) +end + +function _logicalvalue(node::SchemaNode, value; limits::Limits=Limits()) + return _logicalvalue(node.element, value; limits=limits) +end + +function _physicalvalue(kind, element::Metadata.SchemaElement, value, limits::Limits) + ismissing(value) && return missing + if kind isa Union{_TimeLogicalKind,_TimestampLogicalKind,_IntegerLogicalKind} + return _temporalphysicalvalue(kind, element, value) + elseif kind === :string + value isa AbstractString || + throw(ArgumentError("STRING column $(repr(element.name)) contains a non-string value")) + return _toparquetstring(value, limits) + elseif kind === :date + value isa Dates.Date || + throw(ArgumentError("DATE column $(repr(element.name)) contains a non-Date value")) + return _toparquetdate(value) + elseif kind === :decimal + return _decimalphysicalvalue(kind, element, value, limits) + end + converted = _binaryphysicalvalue(kind, element, value, limits) + converted === nothing && throw(ArgumentError( + "unsupported logical annotation $kind on $(repr(element.name))")) + return converted +end + +function _physicalvalue(element::Metadata.SchemaElement, value; limits::Limits=Limits()) + kind = _logicalkind(element) + kind === nothing && return value + return _physicalvalue(kind, element, value, limits) +end + +function _physicalvalue(node::SchemaNode, value; limits::Limits=Limits()) + return _physicalvalue(node.element, value; limits=limits) +end + +function _convertedvector(::Type{T}, values::AbstractVector) where {T} + U = Missing <: eltype(values) ? Union{Missing,T} : T + return Vector{U}(undef, length(values)) +end + +function _logicalvalues(element::Metadata.SchemaElement, values::AbstractVector; + limits::Limits=Limits()) + kind = _logicalkind(element) + kind === nothing && return values + _checklimit(:container_elements, length(values), limits.max_container_elements) + kind === :decimal && _preflightdecimalconversion(element, limits) + physical = _physicaleltype(element.type_) + T = _logicaleltype(element, physical) + output = _convertedvector(T, values) + for (index, value) in enumerate(values) + output[index] = _logicalvalue(kind, element, value, limits) + end + return output +end + +function _logicalvalues(node::SchemaNode, values::AbstractVector; limits::Limits=Limits()) + return _logicalvalues(node.element, values; limits=limits) +end + +function _physicalvalues(element::Metadata.SchemaElement, values::AbstractVector; + limits::Limits=Limits()) + kind = _logicalkind(element) + kind === nothing && return values + _checklimit(:container_elements, length(values), limits.max_container_elements) + kind === :decimal && _preflightdecimalconversion(element, limits) + T = _physicaleltype(element.type_) + output = _convertedvector(T, values) + for (index, value) in enumerate(values) + output[index] = _physicalvalue(kind, element, value, limits) + end + return output +end + +function _physicalvalues(node::SchemaNode, values::AbstractVector; limits::Limits=Limits()) + return _physicalvalues(node.element, values; limits=limits) +end diff --git a/src/logical_binary.jl b/src/logical_binary.jl new file mode 100644 index 0000000..b2c7e99 --- /dev/null +++ b/src/logical_binary.jl @@ -0,0 +1,428 @@ +import UUIDs + +"""An encoded JSON document whose UTF-8 bytes are preserved exactly.""" +struct JSONValue + bytes::Base.CodeUnits{UInt8,String} + + function JSONValue(bytes::AbstractVector{UInt8}; limits::Limits=Limits()) + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + owned = collect(bytes) + _validatejson(owned, limits, ArgumentError) + return new(codeunits(String(owned))) + end + + function JSONValue(bytes::Vector{UInt8}, ::Val{:validated}) + return new(codeunits(String(bytes))) + end +end + +"""An encoded BSON document whose bytes are preserved exactly.""" +struct BSONValue + bytes::Base.CodeUnits{UInt8,String} + + function BSONValue(bytes::AbstractVector{UInt8}; limits::Limits=Limits()) + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + owned = collect(bytes) + _validatebson(owned, limits, ArgumentError) + return new(codeunits(String(owned))) + end + + function BSONValue(bytes::Vector{UInt8}, ::Val{:validated}) + return new(codeunits(String(bytes))) + end +end + +"""A Parquet INTERVAL with independent unsigned month, day, and millisecond fields.""" +struct Interval + months::UInt32 + days::UInt32 + milliseconds::UInt32 +end + +function Interval(months::Integer, days::Integer, milliseconds::Integer) + 0 <= months <= typemax(UInt32) || + throw(ArgumentError("INTERVAL months must fit UInt32")) + 0 <= days <= typemax(UInt32) || + throw(ArgumentError("INTERVAL days must fit UInt32")) + 0 <= milliseconds <= typemax(UInt32) || + throw(ArgumentError("INTERVAL milliseconds must fit UInt32")) + return Interval(UInt32(months), UInt32(days), UInt32(milliseconds)) +end + +function Base.:(==)(left::JSONValue, right::JSONValue) + return left.bytes == right.bytes +end + +function Base.isequal(left::JSONValue, right::JSONValue) + return isequal(left.bytes, right.bytes) +end + +function Base.hash(value::JSONValue, seed::UInt) + return hash(value.bytes, hash(:JSONValue, seed)) +end + +function Base.copy(value::JSONValue) + return JSONValue(collect(value.bytes), Val(:validated)) +end + +function Base.:(==)(left::BSONValue, right::BSONValue) + return left.bytes == right.bytes +end + +function Base.isequal(left::BSONValue, right::BSONValue) + return isequal(left.bytes, right.bytes) +end + +function Base.hash(value::BSONValue, seed::UInt) + return hash(value.bytes, hash(:BSONValue, seed)) +end + +function Base.copy(value::BSONValue) + return BSONValue(collect(value.bytes), Val(:validated)) +end + +function Base.:(==)(left::Interval, right::Interval) + return left.months == right.months && left.days == right.days && + left.milliseconds == right.milliseconds +end + +function Base.isequal(left::Interval, right::Interval) + return isequal(left.months, right.months) && isequal(left.days, right.days) && + isequal(left.milliseconds, right.milliseconds) +end + +function Base.hash(value::Interval, seed::UInt) + return hash((value.months, value.days, value.milliseconds), + hash(:Interval, seed)) +end + +function Base.copy(value::Interval) + return value +end + +function _requirefixedlogical(element::Metadata.SchemaElement, width::Int, + annotation::Symbol) + _requirelogicalphysical(element, Metadata.Type.FIXED_LEN_BYTE_ARRAY, annotation) + element.type_length == width && return + throw(FormatError("$annotation annotation on $(repr(element.name)) requires " * + "FIXED_LEN_BYTE_ARRAY length $width, got $(element.type_length)")) +end + +function _requireunknownphysical(element::Metadata.SchemaElement) + element.type_ === nothing && + throw(FormatError("UNKNOWN annotation on $(repr(element.name)) requires " * + "a primitive physical type")) + element.repetition_type == Metadata.FieldRepetitionType.REQUIRED && + throw(FormatError("UNKNOWN annotation on required field " * + "$(repr(element.name)) cannot represent null")) + return +end + +function _binarymodernkind(element::Metadata.SchemaElement, + logical::Metadata.LogicalType) + if logical.ENUM !== nothing + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :ENUM) + return :enum + elseif logical.UNKNOWN !== nothing + _requireunknownphysical(element) + return :unknown + elseif logical.JSON !== nothing + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :JSON) + return :json + elseif logical.BSON !== nothing + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :BSON) + return :bson + elseif logical.UUID !== nothing + _requirefixedlogical(element, 16, :UUID) + return :uuid + elseif logical.FLOAT16 !== nothing + _requirefixedlogical(element, 2, :FLOAT16) + return :float16 + end + return nothing +end + +function _binarylegacykind(element::Metadata.SchemaElement) + converted = element.converted_type + if converted == Metadata.ConvertedType.ENUM + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :ENUM) + return :enum + elseif converted == Metadata.ConvertedType.JSON + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :JSON) + return :json + elseif converted == Metadata.ConvertedType.BSON + _requirelogicalphysical(element, Metadata.Type.BYTE_ARRAY, :BSON) + return :bson + elseif converted == Metadata.ConvertedType.INTERVAL + _requirefixedlogical(element, 12, :INTERVAL) + return :interval + end + return nothing +end + +function _binarylogicalkind(element::Metadata.SchemaElement) + logical = element.logicalType + logical === nothing || return _binarymodernkind(element, logical) + return _binarylegacykind(element) +end + +function _binarylogicalkind(node::SchemaNode) + return _binarylogicalkind(node.element) +end + +function _binarylogicaleltype(kind::Symbol, physical::Type) + kind === :enum && return String + kind === :uuid && return UUIDs.UUID + kind === :float16 && return Float16 + kind === :json && return JSONValue + kind === :bson && return BSONValue + kind === :interval && return Interval + kind === :unknown && return Missing + return nothing +end + +function _binarylogicaleltype(element::Metadata.SchemaElement, physical::Type) + kind = _binarylogicalkind(element) + kind === nothing && return nothing + return _binarylogicaleltype(kind, physical) +end + +function _binarylogicaleltype(node::SchemaNode, physical::Type) + return _binarylogicaleltype(node.element, physical) +end + +function _requirelogicalbytes(value, element::Metadata.SchemaElement, + annotation::Symbol) + value isa AbstractVector{UInt8} && return value + throw(FormatError("$annotation column $(repr(element.name)) contains a non-byte-array value")) +end + +function _fixedlogicalbytes(value, element::Metadata.SchemaElement, width::Int, + annotation::Symbol) + bytes = _requirelogicalbytes(value, element, annotation) + length(bytes) == width || + throw(FormatError("$annotation column $(repr(element.name)) contains " * + "$(length(bytes)) bytes, expected $width")) + return bytes +end + +function _uuidfrombytes(bytes::AbstractVector{UInt8}) + value = UInt128(0) + for byte in bytes + value = (value << 8) | UInt128(byte) + end + return UUIDs.UUID(value) +end + +function _uuidbytes(value::UUIDs.UUID) + raw = UInt128(value) + bytes = Vector{UInt8}(undef, 16) + for index in eachindex(bytes) + shift = 8 * (length(bytes) - index) + bytes[index] = UInt8((raw >> shift) & 0xff) + end + return bytes +end + +function _float16frombytes(bytes::AbstractVector{UInt8}) + bits = UInt16(bytes[1]) | (UInt16(bytes[2]) << 8) + return reinterpret(Float16, bits) +end + +function _float16bytes(value::Float16) + bits = reinterpret(UInt16, value) + return UInt8[UInt8(bits & 0xff), UInt8(bits >> 8)] +end + +function _readuint32le(bytes::AbstractVector{UInt8}, offset::Int) + return UInt32(bytes[offset]) | + (UInt32(bytes[offset + 1]) << 8) | + (UInt32(bytes[offset + 2]) << 16) | + (UInt32(bytes[offset + 3]) << 24) +end + +function _intervalfrombytes(bytes::AbstractVector{UInt8}) + return Interval(_readuint32le(bytes, 1), _readuint32le(bytes, 5), + _readuint32le(bytes, 9)) +end + +function _appenduint32le!(bytes::Vector{UInt8}, value::UInt32) + push!(bytes, UInt8(value & 0xff)) + push!(bytes, UInt8((value >> 8) & 0xff)) + push!(bytes, UInt8((value >> 16) & 0xff)) + push!(bytes, UInt8(value >> 24)) + return +end + +function _intervalbytes(value::Interval) + bytes = UInt8[] + sizehint!(bytes, 12) + _appenduint32le!(bytes, value.months) + _appenduint32le!(bytes, value.days) + _appenduint32le!(bytes, value.milliseconds) + return bytes +end + +function _taggedlogicalvalue(::Type{JSONValue}, value, + element::Metadata.SchemaElement, limits::Limits) + bytes = _requirelogicalbytes(value, element, :JSON) + _validatejson(bytes, limits, FormatError) + return JSONValue(collect(bytes), Val(:validated)) +end + +function _taggedlogicalvalue(::Type{BSONValue}, value, + element::Metadata.SchemaElement, limits::Limits) + bytes = _requirelogicalbytes(value, element, :BSON) + _validatebson(bytes, limits, FormatError) + return BSONValue(collect(bytes), Val(:validated)) +end + +function _binarylogicalvalue(kind::Symbol, element::Metadata.SchemaElement, value, + limits::Limits) + if kind === :unknown + ismissing(value) && return missing + throw(FormatError("UNKNOWN column $(repr(element.name)) contains a non-null value")) + end + ismissing(value) && return missing + if kind === :enum + bytes = _requirelogicalbytes(value, element, :ENUM) + return _fromparquetstring(bytes, limits) + elseif kind === :uuid + return _uuidfrombytes(_fixedlogicalbytes(value, element, 16, :UUID)) + elseif kind === :float16 + return _float16frombytes(_fixedlogicalbytes(value, element, 2, :FLOAT16)) + elseif kind === :json + return _taggedlogicalvalue(JSONValue, value, element, limits) + elseif kind === :bson + return _taggedlogicalvalue(BSONValue, value, element, limits) + elseif kind === :interval + return _intervalfrombytes(_fixedlogicalbytes(value, element, 12, :INTERVAL)) + end + return nothing +end + +function _binarylogicalvalue(element::Metadata.SchemaElement, value; + limits::Limits=Limits()) + kind = _binarylogicalkind(element) + kind === nothing && return nothing + return _binarylogicalvalue(kind, element, value, limits) +end + +function _binarylogicalvalue(node::SchemaNode, value; limits::Limits=Limits()) + return _binarylogicalvalue(node.element, value; limits=limits) +end + +function _taggedphysicalvalue(value::JSONValue, limits::Limits) + _validatejson(value.bytes, limits, ArgumentError) + return collect(value.bytes) +end + +function _taggedphysicalvalue(value::BSONValue, limits::Limits) + _validatebson(value.bytes, limits, ArgumentError) + return collect(value.bytes) +end + +function _binaryphysicalvalue(kind::Symbol, element::Metadata.SchemaElement, value, + limits::Limits) + if kind === :unknown + ismissing(value) && return missing + throw(ArgumentError("UNKNOWN column $(repr(element.name)) contains a non-null value")) + end + ismissing(value) && return missing + if kind === :enum + value isa AbstractString || + throw(ArgumentError("ENUM column $(repr(element.name)) contains a non-string value")) + return _toparquetstring(value, limits) + elseif kind === :uuid + value isa UUIDs.UUID || + throw(ArgumentError("UUID column $(repr(element.name)) contains a non-UUID value")) + return _uuidbytes(value) + elseif kind === :float16 + value isa Float16 || + throw(ArgumentError("FLOAT16 column $(repr(element.name)) contains " * + "a non-Float16 value")) + return _float16bytes(value) + elseif kind === :json + value isa JSONValue || + throw(ArgumentError("JSON column $(repr(element.name)) contains an untagged value")) + return _taggedphysicalvalue(value, limits) + elseif kind === :bson + value isa BSONValue || + throw(ArgumentError("BSON column $(repr(element.name)) contains an untagged value")) + return _taggedphysicalvalue(value, limits) + elseif kind === :interval + value isa Interval || + throw(ArgumentError("INTERVAL column $(repr(element.name)) contains " * + "a non-Interval value")) + return _intervalbytes(value) + end + return nothing +end + +function _binaryphysicalvalue(element::Metadata.SchemaElement, value; + limits::Limits=Limits()) + kind = _binarylogicalkind(element) + kind === nothing && return nothing + return _binaryphysicalvalue(kind, element, value, limits) +end + +function _binaryphysicalvalue(node::SchemaNode, value; limits::Limits=Limits()) + return _binaryphysicalvalue(node.element, value; limits=limits) +end + +function _unknownlogicalvalues(element::Metadata.SchemaElement, + values::AbstractVector, limits::Limits) + _checklimit(:container_elements, length(values), limits.max_container_elements) + output = Vector{Missing}(undef, length(values)) + for (index, value) in enumerate(values) + output[index] = _binarylogicalvalue(:unknown, element, value, limits) + end + return output +end + +function _unknownphysicalvalues(element::Metadata.SchemaElement, + values::AbstractVector, limits::Limits) + _checklimit(:container_elements, length(values), limits.max_container_elements) + output = Vector{Missing}(undef, length(values)) + for (index, value) in enumerate(values) + output[index] = _binaryphysicalvalue(:unknown, element, value, limits) + end + return output +end + +function _binarylogicalvalues(element::Metadata.SchemaElement, + values::AbstractVector; limits::Limits=Limits()) + kind = _binarylogicalkind(element) + kind === nothing && return nothing + kind === :unknown && return _unknownlogicalvalues(element, values, limits) + _checklimit(:container_elements, length(values), limits.max_container_elements) + T = _binarylogicaleltype(kind, eltype(values)) + output = _convertedvector(T, values) + for (index, value) in enumerate(values) + output[index] = _binarylogicalvalue(kind, element, value, limits) + end + return output +end + +function _binarylogicalvalues(node::SchemaNode, values::AbstractVector; + limits::Limits=Limits()) + return _binarylogicalvalues(node.element, values; limits=limits) +end + +function _binaryphysicalvalues(element::Metadata.SchemaElement, + values::AbstractVector; limits::Limits=Limits()) + kind = _binarylogicalkind(element) + kind === nothing && return nothing + kind === :unknown && return _unknownphysicalvalues(element, values, limits) + _checklimit(:container_elements, length(values), limits.max_container_elements) + output = _convertedvector(Vector{UInt8}, values) + for (index, value) in enumerate(values) + output[index] = _binaryphysicalvalue(kind, element, value, limits) + end + return output +end + +function _binaryphysicalvalues(node::SchemaNode, values::AbstractVector; + limits::Limits=Limits()) + return _binaryphysicalvalues(node.element, values; limits=limits) +end diff --git a/src/logical_bson.jl b/src/logical_bson.jl new file mode 100644 index 0000000..03427c7 --- /dev/null +++ b/src/logical_bson.jl @@ -0,0 +1,196 @@ +mutable struct _BSONValidator{B<:AbstractVector{UInt8}} + bytes::B + elements::Int64 + limits::Limits +end + +function _bsonfail(::Type{E}, message::String) where {E<:Exception} + throw(E("invalid BSON: $message")) +end + +function _bsonrequire(position::Int, count::Int, stop::Int, + ::Type{E}) where {E<:Exception} + count <= stop - position + 1 && return + _bsonfail(E, "input ends inside a value") +end + +function _bsonint32(bytes::AbstractVector{UInt8}, position::Int) + raw = UInt32(bytes[position]) | + (UInt32(bytes[position + 1]) << 8) | + (UInt32(bytes[position + 2]) << 16) | + (UInt32(bytes[position + 3]) << 24) + return reinterpret(Int32, raw) +end + +function _bsoncstring(validator::_BSONValidator, position::Int, stop::Int, + ::Type{E}) where {E<:Exception} + start = position + while position <= stop && !iszero(validator.bytes[position]) + position += 1 + end + position <= stop || _bsonfail(E, "unterminated cstring") + value = @view validator.bytes[start:(position - 1)] + isvalid(String, value) || _bsonfail(E, "cstring is not valid UTF-8") + return position + 1, value +end + +function _bsonstring(validator::_BSONValidator, position::Int, stop::Int, + ::Type{E}) where {E<:Exception} + _bsonrequire(position, 4, stop, E) + length = Int(_bsonint32(validator.bytes, position)) + length > 0 || _bsonfail(E, "string has a nonpositive length") + position += 4 + _bsonrequire(position, length, stop, E) + last = position + length - 1 + iszero(validator.bytes[last]) || _bsonfail(E, "string has no trailing null byte") + value = @view validator.bytes[position:(last - 1)] + isvalid(String, value) || _bsonfail(E, "string is not valid UTF-8") + return last + 1 +end + +function _bsonitem!(validator::_BSONValidator) + requested = validator.elements + 1 + _checklimit(:container_elements, requested, validator.limits.max_container_elements) + validator.elements = requested + return +end + +function _bsonarraykey(key::AbstractVector{UInt8}, index::Int) + isempty(key) && return false + length(key) > 1 && first(key) == 0x30 && return false + value = 0 + for byte in key + 0x30 <= byte <= 0x39 || return false + digit = Int(byte - 0x30) + value <= div(typemax(Int) - digit, 10) || return false + value = 10 * value + digit + end + return value == index +end + +function _bsonregexoptions(options::AbstractVector{UInt8}) + previous = UInt8(0) + for option in options + option in (0x69, 0x6c, 0x6d, 0x73, 0x75, 0x78) || return false + option > previous || return false + previous = option + end + return true +end + +function _bsonbinary(validator::_BSONValidator, position::Int, stop::Int, + ::Type{E}) where {E<:Exception} + _bsonrequire(position, 5, stop, E) + length = Int(_bsonint32(validator.bytes, position)) + length >= 0 || _bsonfail(E, "binary value has a negative length") + subtype = validator.bytes[position + 4] + (subtype <= 0x09 || subtype >= 0x80) || + _bsonfail(E, "binary subtype is reserved") + position += 5 + _bsonrequire(position, length, stop, E) + if subtype == 0x02 + length >= 4 || _bsonfail(E, "old binary subtype omits its inner length") + inner = Int(_bsonint32(validator.bytes, position)) + inner == length - 4 || + _bsonfail(E, "old binary subtype lengths do not agree") + end + return position + length +end + +function _bsonregex(validator::_BSONValidator, position::Int, stop::Int, + ::Type{E}) where {E<:Exception} + position, _ = _bsoncstring(validator, position, stop, E) + position, options = _bsoncstring(validator, position, stop, E) + _bsonregexoptions(options) || + _bsonfail(E, "regular-expression options are invalid or unsorted") + return position +end + +function _bsoncodewithscope(validator::_BSONValidator, position::Int, stop::Int, + depth::Int, ::Type{E}) where {E<:Exception} + start = position + _bsonrequire(position, 4, stop, E) + length = Int(_bsonint32(validator.bytes, position)) + length >= 14 || _bsonfail(E, "code-with-scope value is too short") + length <= stop - start + 1 || _bsonfail(E, "code-with-scope value exceeds its document") + last = start + length - 1 + position = _bsonstring(validator, position + 4, last, E) + position = _bsondocument(validator, position, last, depth + 1, false, E) + position == last + 1 || _bsonfail(E, "code-with-scope length is inconsistent") + return position +end + +function _bsonfixed(position::Int, count::Int, stop::Int, + ::Type{E}) where {E<:Exception} + _bsonrequire(position, count, stop, E) + return position + count +end + +function _bsonelementvalue(validator::_BSONValidator, type::UInt8, position::Int, + stop::Int, depth::Int, ::Type{E}) where {E<:Exception} + type == 0x01 && return _bsonfixed(position, 8, stop, E) + type == 0x02 && return _bsonstring(validator, position, stop, E) + type == 0x03 && return _bsondocument(validator, position, stop, depth + 1, false, E) + type == 0x04 && return _bsondocument(validator, position, stop, depth + 1, true, E) + type == 0x05 && return _bsonbinary(validator, position, stop, E) + type == 0x06 && return position + type == 0x07 && return _bsonfixed(position, 12, stop, E) + if type == 0x08 + _bsonrequire(position, 1, stop, E) + validator.bytes[position] in (0x00, 0x01) || + _bsonfail(E, "Boolean value is not zero or one") + return position + 1 + end + type == 0x09 && return _bsonfixed(position, 8, stop, E) + type == 0x0a && return position + type == 0x0b && return _bsonregex(validator, position, stop, E) + if type == 0x0c + position = _bsonstring(validator, position, stop, E) + return _bsonfixed(position, 12, stop, E) + end + type == 0x0d && return _bsonstring(validator, position, stop, E) + type == 0x0e && return _bsonstring(validator, position, stop, E) + type == 0x0f && return _bsoncodewithscope(validator, position, stop, depth, E) + type == 0x10 && return _bsonfixed(position, 4, stop, E) + type == 0x11 && return _bsonfixed(position, 8, stop, E) + type == 0x12 && return _bsonfixed(position, 8, stop, E) + type == 0x13 && return _bsonfixed(position, 16, stop, E) + type in (0x7f, 0xff) && return position + _bsonfail(E, "unknown element type 0x$(string(type, base=16, pad=2))") +end + +function _bsondocument(validator::_BSONValidator, start::Int, bound::Int, + depth::Int, array::Bool, ::Type{E}) where {E<:Exception} + _checklimit(:metadata_depth, depth, validator.limits.max_metadata_depth) + _bsonrequire(start, 4, bound, E) + length = Int(_bsonint32(validator.bytes, start)) + length >= 5 || _bsonfail(E, "document length is less than five bytes") + length <= bound - start + 1 || _bsonfail(E, "document length exceeds its parent") + stop = start + length - 1 + position = start + 4 + index = 0 + while position < stop + type = validator.bytes[position] + iszero(type) && _bsonfail(E, "document terminates before its declared length") + position += 1 + position, key = _bsoncstring(validator, position, stop - 1, E) + array && !_bsonarraykey(key, index) && + _bsonfail(E, "array keys are not consecutive decimal indexes") + _bsonitem!(validator) + position = _bsonelementvalue(validator, type, position, stop - 1, depth, E) + position <= stop || _bsonfail(E, "element exceeds its document") + index += 1 + end + position == stop || _bsonfail(E, "document length ends inside an element") + iszero(validator.bytes[stop]) || _bsonfail(E, "document has no trailing null byte") + return stop + 1 +end + +function _validatebson(bytes::AbstractVector{UInt8}, limits::Limits, + ::Type{E}) where {E<:Exception} + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + validator = _BSONValidator(bytes, Int64(0), limits) + position = _bsondocument(validator, 1, length(bytes), 1, false, E) + position == length(bytes) + 1 || _bsonfail(E, "trailing bytes after the root document") + return +end diff --git a/src/logical_column.jl b/src/logical_column.jl new file mode 100644 index 0000000..33c0ac7 --- /dev/null +++ b/src/logical_column.jl @@ -0,0 +1,247 @@ +abstract type _ScalarLogicalColumnSpec end + +struct _EnumLogicalColumnSpec <: _ScalarLogicalColumnSpec end + +struct _TimeLogicalColumnSpec <: _ScalarLogicalColumnSpec + unit::UInt8 + adjusted::Bool +end + +struct _TimestampLogicalColumnSpec <: _ScalarLogicalColumnSpec + unit::UInt8 + adjusted::Bool +end + +struct _DecimalLogicalColumnSpec <: _ScalarLogicalColumnSpec + precision::Int32 + scale::Int32 +end + +""" + LogicalColumn(values, logical; unit=nothing, adjusted=nothing, + precision=nothing, scale=nothing) + +Attach explicit Parquet scalar logical metadata to a vector used as a Tables.jl +column. Supported logical types are `:enum`, `:time`, `:timestamp`, and +`:decimal`. TIME and TIMESTAMP require `unit` and `adjusted`; DECIMAL requires +`precision` and `scale`. `adjusted` sets Parquet's `isAdjustedToUTC` flag. +""" +struct LogicalColumn{T,V<:AbstractVector,S<:_ScalarLogicalColumnSpec} <: AbstractVector{T} + values::V + spec::S +end + +function _requirelogicalcolumnunset(kind::Symbol, option::Symbol, value) + value === nothing && return + throw(ArgumentError("$option is not valid for a $kind logical column")) +end + +function _logicalcolumnunit(unit) + unit isa Symbol || throw(ArgumentError( + "logical column unit must be :millis, :micros, or :nanos")) + unit === :millis && return _TEMPORAL_MILLIS + unit === :micros && return _TEMPORAL_MICROS + unit === :nanos && return _TEMPORAL_NANOS + throw(ArgumentError("logical column unit must be :millis, :micros, or :nanos")) +end + +function _logicalcolumnadjusted(adjusted) + adjusted isa Bool || throw(ArgumentError( + "TIME and TIMESTAMP logical columns require adjusted=true or adjusted=false")) + return adjusted +end + +function _logicalcolumnint32(value, option::Symbol) + value isa Integer && !(value isa Bool) || throw(ArgumentError( + "$option must be an integer")) + typemin(Int32) <= value <= typemax(Int32) || throw(ArgumentError( + "$option must fit in Int32")) + return Int32(value) +end + +function _logicalcolumnspec(logical, unit, adjusted, precision, scale) + logical isa Symbol || throw(ArgumentError( + "logical column type must be :enum, :time, :timestamp, or :decimal")) + if logical === :enum + _requirelogicalcolumnunset(logical, :unit, unit) + _requirelogicalcolumnunset(logical, :adjusted, adjusted) + _requirelogicalcolumnunset(logical, :precision, precision) + _requirelogicalcolumnunset(logical, :scale, scale) + return _EnumLogicalColumnSpec() + elseif logical === :time + _requirelogicalcolumnunset(logical, :precision, precision) + _requirelogicalcolumnunset(logical, :scale, scale) + return _TimeLogicalColumnSpec(_logicalcolumnunit(unit), + _logicalcolumnadjusted(adjusted)) + elseif logical === :timestamp + _requirelogicalcolumnunset(logical, :precision, precision) + _requirelogicalcolumnunset(logical, :scale, scale) + return _TimestampLogicalColumnSpec(_logicalcolumnunit(unit), + _logicalcolumnadjusted(adjusted)) + elseif logical === :decimal + _requirelogicalcolumnunset(logical, :unit, unit) + _requirelogicalcolumnunset(logical, :adjusted, adjusted) + decimalprecision = _logicalcolumnint32(precision, :precision) + decimalscale = _logicalcolumnint32(scale, :scale) + decimalprecision > 0 || throw(ArgumentError( + "DECIMAL precision must be positive")) + 0 <= decimalscale <= decimalprecision || throw(ArgumentError( + "DECIMAL scale must be between zero and precision")) + return _DecimalLogicalColumnSpec(decimalprecision, decimalscale) + end + throw(ArgumentError( + "logical column type must be :enum, :time, :timestamp, or :decimal")) +end + +function _logicalcolumncanonicaltype(::_EnumLogicalColumnSpec) + return String +end + +function _logicalcolumncanonicaltype(::_TimeLogicalColumnSpec) + return Dates.Time +end + +function _logicalcolumncanonicaltype(spec::_TimestampLogicalColumnSpec) + if spec.unit == _TEMPORAL_MILLIS + spec.adjusted || return Dates.DateTime + return Timestamp{:millis} + end + spec.unit == _TEMPORAL_MICROS && return Timestamp{:micros} + return Timestamp{:nanos} +end + +function _logicalcolumncanonicaltype(::_DecimalLogicalColumnSpec) + return Decimal +end + +function _validatelogicalcolumnvaluetype(::_EnumLogicalColumnSpec, value_type::Type) + value_type <: AbstractString && return + throw(ArgumentError("ENUM logical columns require string values or missing")) +end + +function _validatelogicalcolumnvaluetype(::_TimeLogicalColumnSpec, value_type::Type) + value_type == Dates.Time && return + throw(ArgumentError("TIME logical columns require Dates.Time values or missing")) +end + +function _validatelogicalcolumnvaluetype(spec::_TimestampLogicalColumnSpec, + value_type::Type) + expected = _logicalcolumncanonicaltype(spec) + value_type == expected && return + throw(ArgumentError("$(_temporalunitname(spec.unit)) TIMESTAMP logical columns " * + "require $expected values or missing")) +end + +function _validatelogicalcolumnvaluetype(::_DecimalLogicalColumnSpec, + value_type::Type) + value_type == Decimal && return + throw(ArgumentError("DECIMAL logical columns require Decimal values or missing")) +end + +function _logicalcolumneltype(values::AbstractVector, spec::_ScalarLogicalColumnSpec) + source_type = eltype(values) + source_type == Any && throw(ArgumentError( + "logical column vectors must have a concrete logical element type")) + source_type == Union{} && throw(ArgumentError( + "logical column vectors must have a logical element type")) + source_type == Missing && + return Union{Missing,_logicalcolumncanonicaltype(spec)} + value_type = Base.nonmissingtype(source_type) + _validatelogicalcolumnvaluetype(spec, value_type) + return source_type +end + +function LogicalColumn(values, logical; unit=nothing, adjusted=nothing, + precision=nothing, scale=nothing) + values isa AbstractVector || throw(ArgumentError( + "LogicalColumn values must be an AbstractVector")) + spec = _logicalcolumnspec(logical, unit, adjusted, precision, scale) + T = _logicalcolumneltype(values, spec) + return LogicalColumn{T,typeof(values),typeof(spec)}(values, spec) +end + +function Base.IndexStyle(::Type{<:LogicalColumn{T,V}}) where {T,V} + return IndexStyle(V) +end + +function Base.size(column::LogicalColumn) + return size(column.values) +end + +function Base.axes(column::LogicalColumn) + return axes(column.values) +end + +function Base.length(column::LogicalColumn) + return length(column.values) +end + +function Base.getindex(column::LogicalColumn, index::Int) + return column.values[index] +end + +function Base.setindex!(column::LogicalColumn, value, index::Int) + column.values[index] = value + return value +end + +function Base.parent(column::LogicalColumn) + return column.values +end + +function Base.copy(column::LogicalColumn{T,V,S}) where {T,V,S} + values = copy(column.values) + return LogicalColumn{T,typeof(values),S}(values, column.spec) +end + +function _logicalcolumnwriteelement(name, ::_EnumLogicalColumnSpec, optional::Bool, + ::Limits) + logical = Metadata.LogicalType(ENUM=Metadata.EnumType()) + return _logicalwriteelement(name, Metadata.Type.BYTE_ARRAY, optional; + logical=logical, converted=Metadata.ConvertedType.ENUM) +end + +function _logicalcolumnwriteelement(name, spec::_TimeLogicalColumnSpec, + optional::Bool, ::Limits) + kind = _TimeLogicalKind(spec.unit, spec.adjusted) + logical = Metadata.LogicalType(TIME=Metadata.TimeType( + isAdjustedToUTC=spec.adjusted, unit=_canonicaltimeunit(spec.unit))) + physical = spec.unit == _TEMPORAL_MILLIS ? Metadata.Type.INT32 : Metadata.Type.INT64 + return _logicalwriteelement(name, physical, optional; logical=logical, + converted=_canonicalconverted(kind)) +end + +function _logicalcolumnwriteelement(name, spec::_TimestampLogicalColumnSpec, + optional::Bool, ::Limits) + kind = _TimestampLogicalKind(spec.unit, spec.adjusted) + logical = Metadata.LogicalType(TIMESTAMP=Metadata.TimestampType( + isAdjustedToUTC=spec.adjusted, unit=_canonicaltimeunit(spec.unit))) + return _logicalwriteelement(name, Metadata.Type.INT64, optional; logical=logical, + converted=_canonicalconverted(kind)) +end + +function _logicalcolumnwriteelement(name, spec::_DecimalLogicalColumnSpec, + optional::Bool, limits::Limits) + if spec.precision <= 9 + physical = Metadata.Type.INT32 + width = nothing + elseif spec.precision <= 18 + physical = Metadata.Type.INT64 + width = nothing + else + physical = Metadata.Type.FIXED_LEN_BYTE_ARRAY + width = _decimalwritewidth(spec.precision, limits) + _checklimit(:decimal_bytes, width, limits.max_decimal_bytes) + end + logical = Metadata.LogicalType(DECIMAL=Metadata.DecimalType( + scale=spec.scale, precision=spec.precision)) + return _logicalwriteelement(name, physical, optional; width=width, + logical=logical, converted=Metadata.ConvertedType.DECIMAL, + scale=spec.scale, precision=spec.precision) +end + +function _writecolumn(name, column::LogicalColumn, limits::Limits) + optional = Missing <: eltype(column) + element = _logicalcolumnwriteelement(name, column.spec, optional, limits) + return _writescalarlogicalcolumn(name, column, element, limits) +end diff --git a/src/logical_decimal.jl b/src/logical_decimal.jl new file mode 100644 index 0000000..3a941d3 --- /dev/null +++ b/src/logical_decimal.jl @@ -0,0 +1,254 @@ +""" + Decimal(unscaled, scale) + +An exact Parquet decimal value equal to `unscaled * 10^(-scale)`. +""" +struct Decimal + unscaled::BigInt + scale::Int32 + function Decimal(unscaled::Integer, scale::Integer) + 0 <= scale <= typemax(Int32) || + throw(ArgumentError("decimal scale must be between 0 and $(typemax(Int32))")) + return new(BigInt(unscaled), Int32(scale)) + end +end + +function Base.:(==)(left::Decimal, right::Decimal) + return left.scale == right.scale && left.unscaled == right.unscaled +end + +function Base.isequal(left::Decimal, right::Decimal) + return isequal(left.scale, right.scale) && isequal(left.unscaled, right.unscaled) +end + +function Base.hash(value::Decimal, seed::UInt) + return hash(value.unscaled, hash(value.scale, hash(:Decimal, seed))) +end + +function Base.copy(value::Decimal) + return Decimal(copy(value.unscaled), value.scale) +end + +function Base.show(io::IO, value::Decimal) + print(io, "Parquet.Decimal(", value.unscaled, ", ", value.scale, ")") + return +end + +function _decimalparameters(element::Metadata.SchemaElement) + logical = element.logicalType + if logical !== nothing + decimal = logical.DECIMAL + decimal === nothing && return nothing + return decimal.precision, decimal.scale + end + element.converted_type == Metadata.ConvertedType.DECIMAL || return nothing + precision = element.precision + precision === nothing && throw(FormatError( + "legacy DECIMAL column $(repr(element.name)) has no precision")) + return precision, something(element.scale, Int32(0)) +end + +function _fixeddecimalprecision(width::Integer) + width > 0 || return 0 + bits = Base.checked_sub(Base.checked_mul(Int64(width), Int64(8)), Int64(1)) + return setprecision(BigFloat, 256) do + return floor(Int64, BigFloat(bits) * log10(BigFloat(2))) + end +end + +function _validatedecimalmetadata(element::Metadata.SchemaElement, + precision::Int32, scale::Int32) + precision > 0 || throw(FormatError( + "DECIMAL column $(repr(element.name)) has nonpositive precision $precision")) + 0 <= scale <= precision || throw(FormatError( + "DECIMAL column $(repr(element.name)) has invalid scale $scale for precision $precision")) + physical = element.type_ + if physical == Metadata.Type.INT32 + precision <= 9 || throw(FormatError( + "INT32 DECIMAL column $(repr(element.name)) has precision $precision above 9")) + elseif physical == Metadata.Type.INT64 + precision <= 18 || throw(FormatError( + "INT64 DECIMAL column $(repr(element.name)) has precision $precision above 18")) + elseif physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY + width = element.type_length + width !== nothing && width > 0 || throw(FormatError( + "fixed DECIMAL column $(repr(element.name)) has no positive width")) + capacity = _fixeddecimalprecision(width) + precision <= capacity || throw(FormatError( + "fixed DECIMAL column $(repr(element.name)) has precision $precision " * + "above its $width-byte capacity $capacity")) + elseif physical != Metadata.Type.BYTE_ARRAY + throw(FormatError("DECIMAL annotation on $(repr(element.name)) requires " * + "INT32, INT64, BYTE_ARRAY, or FIXED_LEN_BYTE_ARRAY physical storage")) + end + return +end + +function _decimallogicalkind(element::Metadata.SchemaElement) + parameters = _decimalparameters(element) + parameters === nothing && return nothing + _validatedecimalmetadata(element, parameters...) + return :decimal +end + +function _decimallogicaleltype(kind::Symbol) + kind === :decimal && return Decimal + return nothing +end + +function _decimaldigits(value::BigInt) + iszero(value) && return 1 + return ndigits(abs(value); base=10) +end + +function _checkdecimalvalue(value::BigInt, precision::Int32, name::String, + error::Type{<:Exception}) + digits = _decimaldigits(value) + digits <= precision && return + message = "DECIMAL column $(repr(name)) contains a $digits-digit value " * + "for precision $precision" + throw(error(message)) +end + +function _contiguousdecimalbytes(bytes::AbstractVector{UInt8}) + bytes isa StridedVector{UInt8} && stride(bytes, 1) == 1 && return bytes + return collect(bytes) +end + +function _importdecimalmagnitude(bytes::AbstractVector{UInt8}) + input = _contiguousdecimalbytes(bytes) + value = BigInt(0) + # Import one-byte words in most-significant-first order. + GC.@preserve input value begin + ccall((:__gmpz_import, Base.GMP.libgmp), Cvoid, + (Ref{BigInt}, Csize_t, Cint, Csize_t, Cint, Csize_t, Ptr{Cvoid}), + value, length(input), 1, 1, 1, 0, pointer(input)) + end + return value +end + +function _fromtwoscomplement(bytes::AbstractVector{UInt8}) + isempty(bytes) && throw(FormatError("DECIMAL byte array is empty")) + value = _importdecimalmagnitude(bytes) + iszero(first(bytes) & 0x80) && return value + return value - (BigInt(1) << (8 * length(bytes))) +end + +function _twoscomplementwidth(value::BigInt) + magnitude = Base.GMP.MPZ.sizeinbase(value, 2) + bits = if value >= 0 + Base.checked_add(magnitude, 1) + else + poweroftwo = Base.GMP.MPZ.scan1(value, 0) == magnitude - 1 + poweroftwo ? magnitude : Base.checked_add(magnitude, 1) + end + return max(1, cld(bits, 8)) +end + +function _exportdecimalmagnitude!(output::Vector{UInt8}, value::BigInt) + iszero(value) && return output + bytecount = cld(Base.GMP.MPZ.sizeinbase(value, 2), 8) + start = length(output) - bytecount + 1 + target = @view output[start:end] + # Export one-byte words in most-significant-first order. + _, written = Base.GMP.MPZ.export!(target, value; order=1, endian=1, nails=0) + written == bytecount || error("GMP exported $written bytes, expected $bytecount") + return output +end + +function _negatetwoscomplement!(output::Vector{UInt8}) + for index in eachindex(output) + @inbounds output[index] = ~output[index] + end + for index in lastindex(output):-1:firstindex(output) + @inbounds output[index] += UInt8(1) + @inbounds iszero(output[index]) || break + end + return output +end + +function _totwoscomplement(value::BigInt, width::Integer) + width > 0 || throw(ArgumentError("DECIMAL byte width must be positive")) + required = _twoscomplementwidth(value) + required <= width || throw(ArgumentError( + "DECIMAL value does not fit in $width signed bytes")) + output = zeros(UInt8, Int(width)) + _exportdecimalmagnitude!(output, value) + value >= 0 && return output + return _negatetwoscomplement!(output) +end + +function _preflightdecimalconversion(element::Metadata.SchemaElement, limits::Limits) + element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY || return + width = element.type_length + width === nothing && throw(FormatError( + "fixed DECIMAL column $(repr(element.name)) has no width")) + _checklimit(:decimal_bytes, width, limits.max_decimal_bytes) + _checklimit(:string_bytes, width, limits.max_string_bytes) + return +end + +function _fromparquetdecimal(element::Metadata.SchemaElement, value, limits::Limits) + precision, scale = something(_decimalparameters(element)) + physical = element.type_ + unscaled = if physical == Metadata.Type.INT32 + value isa Int32 || throw(FormatError( + "INT32 DECIMAL column $(repr(element.name)) contains a non-INT32 value")) + BigInt(value) + elseif physical == Metadata.Type.INT64 + value isa Int64 || throw(FormatError( + "INT64 DECIMAL column $(repr(element.name)) contains a non-INT64 value")) + BigInt(value) + else + value isa AbstractVector{UInt8} || throw(FormatError( + "binary DECIMAL column $(repr(element.name)) contains a non-byte-array value")) + _checklimit(:decimal_bytes, length(value), limits.max_decimal_bytes) + _checklimit(:string_bytes, length(value), limits.max_string_bytes) + physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY && + length(value) != element.type_length && throw(FormatError( + "fixed DECIMAL column $(repr(element.name)) has a value with the wrong width")) + _fromtwoscomplement(value) + end + _checkdecimalvalue(unscaled, precision, element.name, FormatError) + return Decimal(unscaled, scale) +end + +function _toparquetdecimal(element::Metadata.SchemaElement, value, limits::Limits) + value isa Decimal || throw(ArgumentError( + "DECIMAL column $(repr(element.name)) contains a non-Decimal value")) + precision, scale = something(_decimalparameters(element)) + value.scale == scale || throw(ArgumentError( + "DECIMAL column $(repr(element.name)) requires scale $scale, got $(value.scale)")) + physical = element.type_ + if physical == Metadata.Type.INT32 + _checkdecimalvalue(value.unscaled, precision, element.name, ArgumentError) + typemin(Int32) <= value.unscaled <= typemax(Int32) || throw(ArgumentError( + "DECIMAL value does not fit in INT32")) + return Int32(value.unscaled) + elseif physical == Metadata.Type.INT64 + _checkdecimalvalue(value.unscaled, precision, element.name, ArgumentError) + typemin(Int64) <= value.unscaled <= typemax(Int64) || throw(ArgumentError( + "DECIMAL value does not fit in INT64")) + return Int64(value.unscaled) + end + width = physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY ? + Int(element.type_length) : _twoscomplementwidth(value.unscaled) + _checklimit(:decimal_bytes, width, limits.max_decimal_bytes) + _checklimit(:string_bytes, width, limits.max_string_bytes) + _checkdecimalvalue(value.unscaled, precision, element.name, ArgumentError) + return _totwoscomplement(value.unscaled, width) +end + +function _decimallogicalvalue(kind::Symbol, element::Metadata.SchemaElement, value, + limits::Limits) + kind === :decimal || return nothing + ismissing(value) && return missing + return _fromparquetdecimal(element, value, limits) +end + +function _decimalphysicalvalue(kind::Symbol, element::Metadata.SchemaElement, value, + limits::Limits) + kind === :decimal || return nothing + ismissing(value) && return missing + return _toparquetdecimal(element, value, limits) +end diff --git a/src/logical_json.jl b/src/logical_json.jl new file mode 100644 index 0000000..ac09685 --- /dev/null +++ b/src/logical_json.jl @@ -0,0 +1,205 @@ +mutable struct _JSONValidator{B<:AbstractVector{UInt8}} + bytes::B + position::Int + elements::Int64 + limits::Limits +end + +function _jsonfail(::Type{E}, message::String) where {E<:Exception} + throw(E("invalid JSON: $message")) +end + +function _jsonwhitespace(byte::UInt8) + return byte == 0x20 || byte == 0x09 || byte == 0x0a || byte == 0x0d +end + +function _jsonskipwhitespace!(validator::_JSONValidator) + while validator.position <= length(validator.bytes) && + _jsonwhitespace(validator.bytes[validator.position]) + validator.position += 1 + end + return +end + +function _jsonrequire(validator::_JSONValidator, count::Int, ::Type{E}) where {E<:Exception} + count <= length(validator.bytes) - validator.position + 1 && return + _jsonfail(E, "input ends inside a token") +end + +function _jsonhexvalue(byte::UInt8) + 0x30 <= byte <= 0x39 && return Int(byte - 0x30) + 0x41 <= byte <= 0x46 && return Int(byte - 0x41 + 10) + 0x61 <= byte <= 0x66 && return Int(byte - 0x61 + 10) + return -1 +end + +function _jsonunicodeescape!(validator::_JSONValidator, ::Type{E}) where {E<:Exception} + _jsonrequire(validator, 4, E) + for _ in 1:4 + digit = _jsonhexvalue(validator.bytes[validator.position]) + digit >= 0 || _jsonfail(E, "invalid hexadecimal Unicode escape") + validator.position += 1 + end + return +end + +function _jsonstring!(validator::_JSONValidator, ::Type{E}) where {E<:Exception} + validator.bytes[validator.position] == 0x22 || _jsonfail(E, "expected a string") + validator.position += 1 + while validator.position <= length(validator.bytes) + byte = validator.bytes[validator.position] + validator.position += 1 + byte == 0x22 && return + byte < 0x20 && _jsonfail(E, "unescaped control byte in a string") + byte == 0x5c || continue + validator.position <= length(validator.bytes) || + _jsonfail(E, "input ends after a string escape") + escaped = validator.bytes[validator.position] + validator.position += 1 + escaped in (0x22, 0x5c, 0x2f, 0x62, 0x66, 0x6e, 0x72, 0x74) && continue + escaped == 0x75 || _jsonfail(E, "invalid string escape") + _jsonunicodeescape!(validator, E) + end + _jsonfail(E, "unterminated string") +end + +function _jsonliteral!(validator::_JSONValidator, literal::String, + ::Type{E}) where {E<:Exception} + bytes = codeunits(literal) + _jsonrequire(validator, length(bytes), E) + for byte in bytes + validator.bytes[validator.position] == byte || + _jsonfail(E, "invalid literal") + validator.position += 1 + end + return +end + +function _jsondigits!(validator::_JSONValidator) + start = validator.position + while validator.position <= length(validator.bytes) && + 0x30 <= validator.bytes[validator.position] <= 0x39 + validator.position += 1 + end + return validator.position - start +end + +function _jsonnumber!(validator::_JSONValidator, ::Type{E}) where {E<:Exception} + bytes = validator.bytes + bytes[validator.position] == 0x2d && (validator.position += 1) + validator.position <= length(bytes) || _jsonfail(E, "incomplete number") + if bytes[validator.position] == 0x30 + validator.position += 1 + validator.position <= length(bytes) && + 0x30 <= bytes[validator.position] <= 0x39 && + _jsonfail(E, "leading zero in a number") + elseif 0x31 <= bytes[validator.position] <= 0x39 + _jsondigits!(validator) + else + _jsonfail(E, "invalid number") + end + if validator.position <= length(bytes) && bytes[validator.position] == 0x2e + validator.position += 1 + _jsondigits!(validator) > 0 || _jsonfail(E, "fraction has no digits") + end + if validator.position <= length(bytes) && + bytes[validator.position] in (0x65, 0x45) + validator.position += 1 + validator.position <= length(bytes) && + bytes[validator.position] in (0x2b, 0x2d) && (validator.position += 1) + _jsondigits!(validator) > 0 || _jsonfail(E, "exponent has no digits") + end + return +end + +function _jsonitem!(validator::_JSONValidator) + requested = validator.elements + 1 + _checklimit(:container_elements, requested, validator.limits.max_container_elements) + validator.elements = requested + return +end + +function _jsonarray!(validator::_JSONValidator, depth::Int, + ::Type{E}) where {E<:Exception} + _checklimit(:metadata_depth, depth, validator.limits.max_metadata_depth) + validator.position += 1 + _jsonskipwhitespace!(validator) + validator.position <= length(validator.bytes) || _jsonfail(E, "unterminated array") + if validator.bytes[validator.position] == 0x5d + validator.position += 1 + return + end + while true + _jsonitem!(validator) + _jsonvalue!(validator, depth, E) + _jsonskipwhitespace!(validator) + validator.position <= length(validator.bytes) || _jsonfail(E, "unterminated array") + byte = validator.bytes[validator.position] + validator.position += 1 + byte == 0x5d && return + byte == 0x2c || _jsonfail(E, "expected a comma or closing bracket") + _jsonskipwhitespace!(validator) + validator.position <= length(validator.bytes) || _jsonfail(E, "unterminated array") + end +end + +function _jsonobject!(validator::_JSONValidator, depth::Int, + ::Type{E}) where {E<:Exception} + _checklimit(:metadata_depth, depth, validator.limits.max_metadata_depth) + validator.position += 1 + _jsonskipwhitespace!(validator) + validator.position <= length(validator.bytes) || _jsonfail(E, "unterminated object") + if validator.bytes[validator.position] == 0x7d + validator.position += 1 + return + end + while true + validator.bytes[validator.position] == 0x22 || + _jsonfail(E, "object key is not a string") + _jsonstring!(validator, E) + _jsonskipwhitespace!(validator) + validator.position <= length(validator.bytes) && + validator.bytes[validator.position] == 0x3a || + _jsonfail(E, "object key is not followed by a colon") + validator.position += 1 + _jsonskipwhitespace!(validator) + _jsonitem!(validator) + _jsonvalue!(validator, depth, E) + _jsonskipwhitespace!(validator) + validator.position <= length(validator.bytes) || _jsonfail(E, "unterminated object") + byte = validator.bytes[validator.position] + validator.position += 1 + byte == 0x7d && return + byte == 0x2c || _jsonfail(E, "expected a comma or closing brace") + _jsonskipwhitespace!(validator) + validator.position <= length(validator.bytes) || _jsonfail(E, "unterminated object") + end +end + +function _jsonvalue!(validator::_JSONValidator, depth::Int, + ::Type{E}) where {E<:Exception} + validator.position <= length(validator.bytes) || _jsonfail(E, "missing value") + byte = validator.bytes[validator.position] + byte == 0x7b && return _jsonobject!(validator, depth + 1, E) + byte == 0x5b && return _jsonarray!(validator, depth + 1, E) + byte == 0x22 && return _jsonstring!(validator, E) + byte == 0x74 && return _jsonliteral!(validator, "true", E) + byte == 0x66 && return _jsonliteral!(validator, "false", E) + byte == 0x6e && return _jsonliteral!(validator, "null", E) + (byte == 0x2d || 0x30 <= byte <= 0x39) && return _jsonnumber!(validator, E) + _jsonfail(E, "unexpected byte at the start of a value") +end + +function _validatejson(bytes::AbstractVector{UInt8}, limits::Limits, + ::Type{E}) where {E<:Exception} + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + isvalid(String, bytes) || _jsonfail(E, "input is not valid UTF-8") + validator = _JSONValidator(bytes, 1, Int64(0), limits) + _jsonskipwhitespace!(validator) + validator.position <= length(bytes) || _jsonfail(E, "document is empty") + _jsonvalue!(validator, 0, E) + _jsonskipwhitespace!(validator) + validator.position == length(bytes) + 1 || + _jsonfail(E, "trailing bytes after the root value") + return +end diff --git a/src/logical_temporal.jl b/src/logical_temporal.jl new file mode 100644 index 0000000..0a8a625 --- /dev/null +++ b/src/logical_temporal.jl @@ -0,0 +1,438 @@ +import Dates + +const _TEMPORAL_MILLIS = UInt8(1) +const _TEMPORAL_MICROS = UInt8(2) +const _TEMPORAL_NANOS = UInt8(3) +const _MILLIS_PER_DAY = Int64(86_400_000) +const _MICROS_PER_DAY = Int64(86_400_000_000) +const _NANOS_PER_DAY = Int64(86_400_000_000_000) +const _PARQUET_DATETIME_EPOCH = Dates.value(Dates.DateTime(1970, 1, 1)) + +""" + Timestamp(ticks, unit, is_adjusted_to_utc) + +An exact Parquet timestamp. `unit` is `:millis`, `:micros`, or `:nanos`. Millisecond +columns with `isAdjustedToUTC=false` read as `Dates.DateTime` instead; `Timestamp` +carries the UTC adjustment for every other combination. +""" +struct Timestamp{U} + ticks::Int64 + is_adjusted_to_utc::Bool + + function Timestamp(ticks::Int64, unit::Symbol, is_adjusted_to_utc::Bool) + if unit === :millis + # Millisecond timestamps without UTC adjustment are represented as + # Dates.DateTime, so Timestamp{:millis} only ever carries adjusted=true. + is_adjusted_to_utc || throw(ArgumentError( + "millisecond timestamps with isAdjustedToUTC=false are represented " * + "as Dates.DateTime; use a DateTime value instead")) + return new{:millis}(ticks, is_adjusted_to_utc) + end + unit === :micros && return new{:micros}(ticks, is_adjusted_to_utc) + unit === :nanos && return new{:nanos}(ticks, is_adjusted_to_utc) + throw(ArgumentError("Timestamp unit must be :millis, :micros, or :nanos")) + end +end + +function _timestampunit(::Timestamp{:millis}) + return :millis +end + +function _timestampunit(::Timestamp{:micros}) + return :micros +end + +function _timestampunit(::Timestamp{:nanos}) + return :nanos +end + +function Base.:(==)(left::Timestamp, right::Timestamp) + return typeof(left) === typeof(right) && left.ticks == right.ticks && + left.is_adjusted_to_utc == right.is_adjusted_to_utc +end + +function Base.isequal(left::Timestamp, right::Timestamp) + return left == right +end + +function Base.hash(value::Timestamp, hashvalue::UInt) + return hash(value.is_adjusted_to_utc, + hash(value.ticks, hash(_timestampunit(value), hashvalue))) +end + +function Base.show(io::IO, value::Timestamp) + print(io, "Timestamp(", value.ticks, ", :", _timestampunit(value), ", ", + value.is_adjusted_to_utc, ")") + return +end + +struct _TimeLogicalKind + unit::UInt8 + is_adjusted_to_utc::Bool +end + +struct _TimestampLogicalKind + unit::UInt8 + is_adjusted_to_utc::Bool +end + +struct _IntegerLogicalKind + bitwidth::UInt8 + signed::Bool +end + +function _temporalunit(unit::Metadata.TimeUnit) + unit.MILLIS !== nothing && return _TEMPORAL_MILLIS + unit.MICROS !== nothing && return _TEMPORAL_MICROS + unit.NANOS !== nothing && return _TEMPORAL_NANOS + return nothing +end + +function _requiretemporalunit(element::Metadata.SchemaElement, + annotation::Symbol, unit::Metadata.TimeUnit) + value = _temporalunit(unit) + value !== nothing && return value + isempty(unit.unknown_fields) && throw(FormatError( + "$annotation annotation on $(repr(element.name)) has no time unit")) + throw(UnsupportedFeatureError( + "$annotation annotation on $(repr(element.name)) uses an unknown time unit")) +end + +function _temporalunitname(unit::UInt8) + unit == _TEMPORAL_MILLIS && return :millis + unit == _TEMPORAL_MICROS && return :micros + unit == _TEMPORAL_NANOS && return :nanos + throw(ArgumentError("unknown temporal unit code $unit")) +end + +function _validatetimephysical(element::Metadata.SchemaElement, unit::UInt8) + expected = unit == _TEMPORAL_MILLIS ? Metadata.Type.INT32 : Metadata.Type.INT64 + _requirelogicalphysical(element, expected, :TIME) + return +end + +function _validateintegerphysical(element::Metadata.SchemaElement, + kind::_IntegerLogicalKind) + expected = kind.bitwidth == 64 ? Metadata.Type.INT64 : Metadata.Type.INT32 + _requirelogicalphysical(element, expected, :INTEGER) + return +end + +function _modernintegerkind(element::Metadata.SchemaElement, integer::Metadata.IntType) + width = Int(integer.bitWidth) + width in (8, 16, 32, 64) || + throw(FormatError("INTEGER annotation on $(repr(element.name)) has invalid " * + "bit width $width")) + kind = _IntegerLogicalKind(UInt8(width), integer.isSigned) + _validateintegerphysical(element, kind) + return kind +end + +function _legacyintegerkind(converted::Metadata.ConvertedType.T) + converted == Metadata.ConvertedType.INT_8 && return _IntegerLogicalKind(8, true) + converted == Metadata.ConvertedType.INT_16 && return _IntegerLogicalKind(16, true) + converted == Metadata.ConvertedType.INT_32 && return _IntegerLogicalKind(32, true) + converted == Metadata.ConvertedType.INT_64 && return _IntegerLogicalKind(64, true) + converted == Metadata.ConvertedType.UINT_8 && return _IntegerLogicalKind(8, false) + converted == Metadata.ConvertedType.UINT_16 && return _IntegerLogicalKind(16, false) + converted == Metadata.ConvertedType.UINT_32 && return _IntegerLogicalKind(32, false) + converted == Metadata.ConvertedType.UINT_64 && return _IntegerLogicalKind(64, false) + return nothing +end + +function _moderntemporallogicalkind(element::Metadata.SchemaElement, + logical::Metadata.LogicalType) + if logical.TIME !== nothing + unit = _requiretemporalunit(element, :TIME, logical.TIME.unit) + _validatetimephysical(element, unit) + return _TimeLogicalKind(unit, logical.TIME.isAdjustedToUTC) + elseif logical.TIMESTAMP !== nothing + unit = _requiretemporalunit(element, :TIMESTAMP, logical.TIMESTAMP.unit) + _requirelogicalphysical(element, Metadata.Type.INT64, :TIMESTAMP) + return _TimestampLogicalKind(unit, logical.TIMESTAMP.isAdjustedToUTC) + elseif logical.INTEGER !== nothing + return _modernintegerkind(element, logical.INTEGER) + end + return nothing +end + +function _legacytemporallogicalkind(element::Metadata.SchemaElement, + converted::Metadata.ConvertedType.T) + if converted == Metadata.ConvertedType.TIME_MILLIS + _requirelogicalphysical(element, Metadata.Type.INT32, :TIME_MILLIS) + return _TimeLogicalKind(_TEMPORAL_MILLIS, true) + elseif converted == Metadata.ConvertedType.TIME_MICROS + _requirelogicalphysical(element, Metadata.Type.INT64, :TIME_MICROS) + return _TimeLogicalKind(_TEMPORAL_MICROS, true) + elseif converted == Metadata.ConvertedType.TIMESTAMP_MILLIS + _requirelogicalphysical(element, Metadata.Type.INT64, :TIMESTAMP_MILLIS) + return _TimestampLogicalKind(_TEMPORAL_MILLIS, true) + elseif converted == Metadata.ConvertedType.TIMESTAMP_MICROS + _requirelogicalphysical(element, Metadata.Type.INT64, :TIMESTAMP_MICROS) + return _TimestampLogicalKind(_TEMPORAL_MICROS, true) + end + kind = _legacyintegerkind(converted) + kind === nothing && return nothing + _validateintegerphysical(element, kind) + return kind +end + +function _temporallogicalkind(element::Metadata.SchemaElement) + logical = element.logicalType + logical !== nothing && return _moderntemporallogicalkind(element, logical) + converted = element.converted_type + converted === nothing && return nothing + return _legacytemporallogicalkind(element, converted) +end + +function _temporallogicalkind(node::SchemaNode) + return _temporallogicalkind(node.element) +end + +function _integerjuliatype(kind::_IntegerLogicalKind) + if kind.signed + kind.bitwidth == 8 && return Int8 + kind.bitwidth == 16 && return Int16 + kind.bitwidth == 32 && return Int32 + return Int64 + end + kind.bitwidth == 8 && return UInt8 + kind.bitwidth == 16 && return UInt16 + kind.bitwidth == 32 && return UInt32 + return UInt64 +end + +function _temporaljuliatype(::_TimeLogicalKind) + return Dates.Time +end + +function _temporaljuliatype(kind::_TimestampLogicalKind) + if kind.unit == _TEMPORAL_MILLIS + kind.is_adjusted_to_utc || return Dates.DateTime + return Timestamp{:millis} + end + kind.unit == _TEMPORAL_MICROS && return Timestamp{:micros} + return Timestamp{:nanos} +end + +function _temporaljuliatype(kind::_IntegerLogicalKind) + return _integerjuliatype(kind) +end + +function _temporallogicaleltype(element::Metadata.SchemaElement, physical::Type) + kind = _temporallogicalkind(element) + kind === nothing && return physical + return _temporaljuliatype(kind) +end + +function _temporallogicaleltype(node::SchemaNode, physical::Type) + return _temporallogicaleltype(node.element, physical) +end + +function _timeparameters(unit::UInt8) + unit == _TEMPORAL_MILLIS && return _MILLIS_PER_DAY, Int64(1_000_000) + unit == _TEMPORAL_MICROS && return _MICROS_PER_DAY, Int64(1_000) + return _NANOS_PER_DAY, Int64(1) +end + +function _fromparquettime(kind::_TimeLogicalKind, value, element::Metadata.SchemaElement) + expected = kind.unit == _TEMPORAL_MILLIS ? Int32 : Int64 + value isa expected || + throw(FormatError("TIME column $(repr(element.name)) contains a non-$expected value")) + ticks = Int64(value) + limit, scale = _timeparameters(kind.unit) + 0 <= ticks < limit || + throw(FormatError("TIME column $(repr(element.name)) is outside one day")) + return Dates.Time(Dates.Nanosecond(ticks * scale)) +end + +function _toparquettime(kind::_TimeLogicalKind, value, element::Metadata.SchemaElement) + value isa Dates.Time || + throw(ArgumentError("TIME column $(repr(element.name)) contains a non-Time value")) + nanoseconds = Dates.value(value) + _, scale = _timeparameters(kind.unit) + rem(nanoseconds, scale) == 0 || + throw(ArgumentError("TIME column $(repr(element.name)) loses precision at " * + "$(_temporalunitname(kind.unit)) resolution")) + ticks = div(nanoseconds, scale) + return kind.unit == _TEMPORAL_MILLIS ? Int32(ticks) : Int64(ticks) +end + +function _fromparquetdatetime(value::Int64) + ordinal = try + Base.checked_add(_PARQUET_DATETIME_EPOCH, value) + catch err + err isa OverflowError || rethrow() + throw(FormatError("millisecond TIMESTAMP $value overflows Dates.DateTime")) + end + return Dates.DateTime(Dates.UTM(ordinal)) +end + +function _toparquetdatetime(value::Dates.DateTime) + ticks = try + Base.checked_sub(Dates.value(value), _PARQUET_DATETIME_EPOCH) + catch err + err isa OverflowError || rethrow() + throw(ArgumentError("TIMESTAMP value $(repr(value)) is outside the Int64 range")) + end + return ticks +end + +function _fromparquettimestamp(kind::_TimestampLogicalKind, value, + element::Metadata.SchemaElement) + value isa Int64 || + throw(FormatError("TIMESTAMP column $(repr(element.name)) contains a non-Int64 value")) + kind.unit == _TEMPORAL_MILLIS && !kind.is_adjusted_to_utc && + return _fromparquetdatetime(value) + return Timestamp(value, _temporalunitname(kind.unit), kind.is_adjusted_to_utc) +end + +function _toparquettimestamp(kind::_TimestampLogicalKind, value, + element::Metadata.SchemaElement) + if kind.unit == _TEMPORAL_MILLIS && !kind.is_adjusted_to_utc + value isa Dates.DateTime || + throw(ArgumentError("millisecond TIMESTAMP column $(repr(element.name)) " * + "contains a non-DateTime value")) + return _toparquetdatetime(value) + end + value isa Timestamp || + throw(ArgumentError("TIMESTAMP column $(repr(element.name)) contains a " * + "non-Timestamp value")) + _timestampunit(value) == _temporalunitname(kind.unit) || + throw(ArgumentError("TIMESTAMP column $(repr(element.name)) has the wrong unit")) + value.is_adjusted_to_utc == kind.is_adjusted_to_utc || + throw(ArgumentError("TIMESTAMP column $(repr(element.name)) has the wrong UTC adjustment")) + return value.ticks +end + +function _checkednarrow(::Type{T}, value::Integer, element::Metadata.SchemaElement, + annotation::AbstractString) where {T<:Integer} + typemin(T) <= value <= typemax(T) || + throw(FormatError("$annotation column $(repr(element.name)) value $value is out of range")) + return T(value) +end + +function _fromparquetinteger(kind::_IntegerLogicalKind, value, + element::Metadata.SchemaElement) + physical = kind.bitwidth == 64 ? Int64 : Int32 + value isa physical || + throw(FormatError("INTEGER column $(repr(element.name)) contains a non-$physical value")) + T = _integerjuliatype(kind) + kind.signed && return _checkednarrow(T, value, element, "INTEGER") + kind.bitwidth == 32 && return reinterpret(UInt32, value) + kind.bitwidth == 64 && return reinterpret(UInt64, value) + return _checkednarrow(T, value, element, "unsigned INTEGER") +end + +function _toparquetinteger(kind::_IntegerLogicalKind, value, + element::Metadata.SchemaElement) + T = _integerjuliatype(kind) + value isa T || + throw(ArgumentError("INTEGER column $(repr(element.name)) contains a non-$T value")) + kind.bitwidth == 64 && kind.signed && return Int64(value) + kind.bitwidth == 64 && return reinterpret(Int64, value) + kind.bitwidth == 32 && kind.signed && return Int32(value) + kind.bitwidth == 32 && return reinterpret(Int32, value) + return Int32(value) +end + +function _temporallogicalvalue(kind::_TimeLogicalKind, element::Metadata.SchemaElement, value) + ismissing(value) && return missing + return _fromparquettime(kind, value, element) +end + +function _temporallogicalvalue(kind::_TimestampLogicalKind, + element::Metadata.SchemaElement, value) + ismissing(value) && return missing + return _fromparquettimestamp(kind, value, element) +end + +function _temporallogicalvalue(kind::_IntegerLogicalKind, + element::Metadata.SchemaElement, value) + ismissing(value) && return missing + return _fromparquetinteger(kind, value, element) +end + +function _temporallogicalvalue(element::Metadata.SchemaElement, value) + kind = _temporallogicalkind(element) + kind === nothing && return value + return _temporallogicalvalue(kind, element, value) +end + +function _temporallogicalvalue(node::SchemaNode, value) + return _temporallogicalvalue(node.element, value) +end + +function _temporalphysicalvalue(kind::_TimeLogicalKind, + element::Metadata.SchemaElement, value) + ismissing(value) && return missing + return _toparquettime(kind, value, element) +end + +function _temporalphysicalvalue(kind::_TimestampLogicalKind, + element::Metadata.SchemaElement, value) + ismissing(value) && return missing + return _toparquettimestamp(kind, value, element) +end + +function _temporalphysicalvalue(kind::_IntegerLogicalKind, + element::Metadata.SchemaElement, value) + ismissing(value) && return missing + return _toparquetinteger(kind, value, element) +end + +function _temporalphysicalvalue(element::Metadata.SchemaElement, value) + kind = _temporallogicalkind(element) + kind === nothing && return value + return _temporalphysicalvalue(kind, element, value) +end + +function _temporalphysicalvalue(node::SchemaNode, value) + return _temporalphysicalvalue(node.element, value) +end + +function _temporallogicalvalues(element::Metadata.SchemaElement, + values::AbstractVector; limits::Limits=Limits()) + kind = _temporallogicalkind(element) + kind === nothing && return values + _checklimit(:container_elements, length(values), limits.max_container_elements) + output = _convertedvector(_temporaljuliatype(kind), values) + for (index, value) in enumerate(values) + output[index] = _temporallogicalvalue(kind, element, value) + end + return output +end + +function _temporallogicalvalues(node::SchemaNode, values::AbstractVector; + limits::Limits=Limits()) + return _temporallogicalvalues(node.element, values; limits=limits) +end + +function _temporalphysicaltype(kind::_TimeLogicalKind) + return kind.unit == _TEMPORAL_MILLIS ? Int32 : Int64 +end + +function _temporalphysicaltype(::_TimestampLogicalKind) + return Int64 +end + +function _temporalphysicaltype(kind::_IntegerLogicalKind) + return kind.bitwidth == 64 ? Int64 : Int32 +end + +function _temporalphysicalvalues(element::Metadata.SchemaElement, + values::AbstractVector; limits::Limits=Limits()) + kind = _temporallogicalkind(element) + kind === nothing && return values + _checklimit(:container_elements, length(values), limits.max_container_elements) + output = _convertedvector(_temporalphysicaltype(kind), values) + for (index, value) in enumerate(values) + output[index] = _temporalphysicalvalue(kind, element, value) + end + return output +end + +function _temporalphysicalvalues(node::SchemaNode, values::AbstractVector; + limits::Limits=Limits()) + return _temporalphysicalvalues(node.element, values; limits=limits) +end diff --git a/src/metadata/parquet.jl b/src/metadata/parquet.jl new file mode 100644 index 0000000..f0b015a --- /dev/null +++ b/src/metadata/parquet.jl @@ -0,0 +1,5756 @@ +# Generated by thrift/generate.jl from thrift/parquet.thrift. Do not edit by hand. +# Source: apache/parquet-format 2.13.0 (c47e2a66e88943fc46fde1b028a9432f14fdf5c0) +# IDL: 50913 bytes, FNV-1a 64 0x9ce6392d7707874a +module Metadata + +import ..Thrift + +module Type + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const BOOLEAN = T(0) + +const INT32 = T(1) + +const INT64 = T(2) + +const INT96 = T(3) + +const FLOAT = T(4) + +const DOUBLE = T(5) + +const BYTE_ARRAY = T(6) + +const FIXED_LEN_BYTE_ARRAY = T(7) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :BOOLEAN), (Int32(1), :INT32), (Int32(2), :INT64), (Int32(3), :INT96), (Int32(4), :FLOAT), (Int32(5), :DOUBLE), (Int32(6), :BYTE_ARRAY), (Int32(7), :FIXED_LEN_BYTE_ARRAY)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +module ConvertedType + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const UTF8 = T(0) + +const MAP = T(1) + +const MAP_KEY_VALUE = T(2) + +const LIST = T(3) + +const ENUM = T(4) + +const DECIMAL = T(5) + +const DATE = T(6) + +const TIME_MILLIS = T(7) + +const TIME_MICROS = T(8) + +const TIMESTAMP_MILLIS = T(9) + +const TIMESTAMP_MICROS = T(10) + +const UINT_8 = T(11) + +const UINT_16 = T(12) + +const UINT_32 = T(13) + +const UINT_64 = T(14) + +const INT_8 = T(15) + +const INT_16 = T(16) + +const INT_32 = T(17) + +const INT_64 = T(18) + +const JSON = T(19) + +const BSON = T(20) + +const INTERVAL = T(21) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :UTF8), (Int32(1), :MAP), (Int32(2), :MAP_KEY_VALUE), (Int32(3), :LIST), (Int32(4), :ENUM), (Int32(5), :DECIMAL), (Int32(6), :DATE), (Int32(7), :TIME_MILLIS), (Int32(8), :TIME_MICROS), (Int32(9), :TIMESTAMP_MILLIS), (Int32(10), :TIMESTAMP_MICROS), (Int32(11), :UINT_8), (Int32(12), :UINT_16), (Int32(13), :UINT_32), (Int32(14), :UINT_64), (Int32(15), :INT_8), (Int32(16), :INT_16), (Int32(17), :INT_32), (Int32(18), :INT_64), (Int32(19), :JSON), (Int32(20), :BSON), (Int32(21), :INTERVAL)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +module FieldRepetitionType + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const REQUIRED = T(0) + +const OPTIONAL = T(1) + +const REPEATED = T(2) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :REQUIRED), (Int32(1), :OPTIONAL), (Int32(2), :REPEATED)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +# Thrift struct SizeStatistics +Base.@kwdef struct SizeStatistics + unencoded_byte_array_data_bytes::Union{Nothing, Int64} = nothing # 1: optional i64 unencoded_byte_array_data_bytes + repetition_level_histogram::Union{Nothing, Vector{Int64}} = nothing # 2: optional list repetition_level_histogram + definition_level_histogram::Union{Nothing, Vector{Int64}} = nothing # 3: optional list definition_level_histogram + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::SizeStatistics, b::SizeStatistics) + return a.unencoded_byte_array_data_bytes == b.unencoded_byte_array_data_bytes && a.repetition_level_histogram == b.repetition_level_histogram && a.definition_level_histogram == b.definition_level_histogram && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::SizeStatistics, b::SizeStatistics) + return isequal(a.unencoded_byte_array_data_bytes, b.unencoded_byte_array_data_bytes) && isequal(a.repetition_level_histogram, b.repetition_level_histogram) && isequal(a.definition_level_histogram, b.definition_level_histogram) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::SizeStatistics, h::UInt) + h = hash(:SizeStatistics, h) + h = hash(x.unencoded_byte_array_data_bytes, h) + h = hash(x.repetition_level_histogram, h) + h = hash(x.definition_level_histogram, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{SizeStatistics}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{SizeStatistics}) + return Thrift.decode(r, SizeStatistics) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::SizeStatistics) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{SizeStatistics}) + Thrift.enter!(r) + f_unencoded_byte_array_data_bytes = nothing + f_repetition_level_histogram = nothing + f_definition_level_histogram = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I64 + f_unencoded_byte_array_data_bytes = Thrift.readi64(r) + elseif id == Int16(2) && ty == Thrift.LIST + value_repetition_level_histogram = Thrift.readlist(r, Int64) + if value_repetition_level_histogram === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_repetition_level_histogram = value_repetition_level_histogram + end + elseif id == Int16(3) && ty == Thrift.LIST + value_definition_level_histogram = Thrift.readlist(r, Int64) + if value_definition_level_histogram === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_definition_level_histogram = value_definition_level_histogram + end + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return SizeStatistics(f_unencoded_byte_array_data_bytes, f_repetition_level_histogram, f_definition_level_histogram, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::SizeStatistics) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_unencoded_byte_array_data_bytes = x.unencoded_byte_array_data_bytes + if value_unencoded_byte_array_data_bytes !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I64) + Thrift.writei64!(w, value_unencoded_byte_array_data_bytes) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_repetition_level_histogram = x.repetition_level_histogram + if value_repetition_level_histogram !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.LIST) + Thrift.writelist!(w, value_repetition_level_histogram) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_definition_level_histogram = x.definition_level_histogram + if value_definition_level_histogram !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.LIST) + Thrift.writelist!(w, value_definition_level_histogram) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct BoundingBox +Base.@kwdef struct BoundingBox + xmin::Float64 # 1: required double xmin + xmax::Float64 # 2: required double xmax + ymin::Float64 # 3: required double ymin + ymax::Float64 # 4: required double ymax + zmin::Union{Nothing, Float64} = nothing # 5: optional double zmin + zmax::Union{Nothing, Float64} = nothing # 6: optional double zmax + mmin::Union{Nothing, Float64} = nothing # 7: optional double mmin + mmax::Union{Nothing, Float64} = nothing # 8: optional double mmax + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::BoundingBox, b::BoundingBox) + return a.xmin == b.xmin && a.xmax == b.xmax && a.ymin == b.ymin && a.ymax == b.ymax && a.zmin == b.zmin && a.zmax == b.zmax && a.mmin == b.mmin && a.mmax == b.mmax && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::BoundingBox, b::BoundingBox) + return isequal(a.xmin, b.xmin) && isequal(a.xmax, b.xmax) && isequal(a.ymin, b.ymin) && isequal(a.ymax, b.ymax) && isequal(a.zmin, b.zmin) && isequal(a.zmax, b.zmax) && isequal(a.mmin, b.mmin) && isequal(a.mmax, b.mmax) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::BoundingBox, h::UInt) + h = hash(:BoundingBox, h) + h = hash(x.xmin, h) + h = hash(x.xmax, h) + h = hash(x.ymin, h) + h = hash(x.ymax, h) + h = hash(x.zmin, h) + h = hash(x.zmax, h) + h = hash(x.mmin, h) + h = hash(x.mmax, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{BoundingBox}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{BoundingBox}) + return Thrift.decode(r, BoundingBox) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::BoundingBox) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{BoundingBox}) + Thrift.enter!(r) + f_xmin = nothing + f_xmax = nothing + f_ymin = nothing + f_ymax = nothing + f_zmin = nothing + f_zmax = nothing + f_mmin = nothing + f_mmax = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.DOUBLE + f_xmin = Thrift.readdouble(r) + elseif id == Int16(2) && ty == Thrift.DOUBLE + f_xmax = Thrift.readdouble(r) + elseif id == Int16(3) && ty == Thrift.DOUBLE + f_ymin = Thrift.readdouble(r) + elseif id == Int16(4) && ty == Thrift.DOUBLE + f_ymax = Thrift.readdouble(r) + elseif id == Int16(5) && ty == Thrift.DOUBLE + f_zmin = Thrift.readdouble(r) + elseif id == Int16(6) && ty == Thrift.DOUBLE + f_zmax = Thrift.readdouble(r) + elseif id == Int16(7) && ty == Thrift.DOUBLE + f_mmin = Thrift.readdouble(r) + elseif id == Int16(8) && ty == Thrift.DOUBLE + f_mmax = Thrift.readdouble(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_xmin === nothing && Thrift.missingfield(:BoundingBox, :xmin) + f_xmax === nothing && Thrift.missingfield(:BoundingBox, :xmax) + f_ymin === nothing && Thrift.missingfield(:BoundingBox, :ymin) + f_ymax === nothing && Thrift.missingfield(:BoundingBox, :ymax) + return BoundingBox(f_xmin, f_xmax, f_ymin, f_ymax, f_zmin, f_zmax, f_mmin, f_mmax, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::BoundingBox) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.DOUBLE) + Thrift.writedouble!(w, x.xmin) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.DOUBLE) + Thrift.writedouble!(w, x.xmax) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.DOUBLE) + Thrift.writedouble!(w, x.ymin) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.DOUBLE) + Thrift.writedouble!(w, x.ymax) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_zmin = x.zmin + if value_zmin !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.DOUBLE) + Thrift.writedouble!(w, value_zmin) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_zmax = x.zmax + if value_zmax !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.DOUBLE) + Thrift.writedouble!(w, value_zmax) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_mmin = x.mmin + if value_mmin !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.DOUBLE) + Thrift.writedouble!(w, value_mmin) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_mmax = x.mmax + if value_mmax !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.DOUBLE) + Thrift.writedouble!(w, value_mmax) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct GeospatialStatistics +Base.@kwdef struct GeospatialStatistics + bbox::Union{Nothing, BoundingBox} = nothing # 1: optional BoundingBox bbox + geospatial_types::Union{Nothing, Vector{Int32}} = nothing # 2: optional list geospatial_types + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::GeospatialStatistics, b::GeospatialStatistics) + return a.bbox == b.bbox && a.geospatial_types == b.geospatial_types && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::GeospatialStatistics, b::GeospatialStatistics) + return isequal(a.bbox, b.bbox) && isequal(a.geospatial_types, b.geospatial_types) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::GeospatialStatistics, h::UInt) + h = hash(:GeospatialStatistics, h) + h = hash(x.bbox, h) + h = hash(x.geospatial_types, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{GeospatialStatistics}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{GeospatialStatistics}) + return Thrift.decode(r, GeospatialStatistics) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::GeospatialStatistics) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{GeospatialStatistics}) + Thrift.enter!(r) + f_bbox = nothing + f_geospatial_types = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_bbox = Thrift.decode(r, BoundingBox) + elseif id == Int16(2) && ty == Thrift.LIST + value_geospatial_types = Thrift.readlist(r, Int32) + if value_geospatial_types === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_geospatial_types = value_geospatial_types + end + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return GeospatialStatistics(f_bbox, f_geospatial_types, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::GeospatialStatistics) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_bbox = x.bbox + if value_bbox !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_bbox) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_geospatial_types = x.geospatial_types + if value_geospatial_types !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.LIST) + Thrift.writelist!(w, value_geospatial_types) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct Statistics +Base.@kwdef struct Statistics + max::Union{Nothing, Vector{UInt8}} = nothing # 1: optional binary max + min::Union{Nothing, Vector{UInt8}} = nothing # 2: optional binary min + null_count::Union{Nothing, Int64} = nothing # 3: optional i64 null_count + distinct_count::Union{Nothing, Int64} = nothing # 4: optional i64 distinct_count + max_value::Union{Nothing, Vector{UInt8}} = nothing # 5: optional binary max_value + min_value::Union{Nothing, Vector{UInt8}} = nothing # 6: optional binary min_value + is_max_value_exact::Union{Nothing, Bool} = nothing # 7: optional bool is_max_value_exact + is_min_value_exact::Union{Nothing, Bool} = nothing # 8: optional bool is_min_value_exact + nan_count::Union{Nothing, Int64} = nothing # 9: optional i64 nan_count + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::Statistics, b::Statistics) + return a.max == b.max && a.min == b.min && a.null_count == b.null_count && a.distinct_count == b.distinct_count && a.max_value == b.max_value && a.min_value == b.min_value && a.is_max_value_exact == b.is_max_value_exact && a.is_min_value_exact == b.is_min_value_exact && a.nan_count == b.nan_count && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::Statistics, b::Statistics) + return isequal(a.max, b.max) && isequal(a.min, b.min) && isequal(a.null_count, b.null_count) && isequal(a.distinct_count, b.distinct_count) && isequal(a.max_value, b.max_value) && isequal(a.min_value, b.min_value) && isequal(a.is_max_value_exact, b.is_max_value_exact) && isequal(a.is_min_value_exact, b.is_min_value_exact) && isequal(a.nan_count, b.nan_count) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::Statistics, h::UInt) + h = hash(:Statistics, h) + h = hash(x.max, h) + h = hash(x.min, h) + h = hash(x.null_count, h) + h = hash(x.distinct_count, h) + h = hash(x.max_value, h) + h = hash(x.min_value, h) + h = hash(x.is_max_value_exact, h) + h = hash(x.is_min_value_exact, h) + h = hash(x.nan_count, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{Statistics}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{Statistics}) + return Thrift.decode(r, Statistics) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::Statistics) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{Statistics}) + Thrift.enter!(r) + f_max = nothing + f_min = nothing + f_null_count = nothing + f_distinct_count = nothing + f_max_value = nothing + f_min_value = nothing + f_is_max_value_exact = nothing + f_is_min_value_exact = nothing + f_nan_count = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BINARY + f_max = Thrift.readbinary(r) + elseif id == Int16(2) && ty == Thrift.BINARY + f_min = Thrift.readbinary(r) + elseif id == Int16(3) && ty == Thrift.I64 + f_null_count = Thrift.readi64(r) + elseif id == Int16(4) && ty == Thrift.I64 + f_distinct_count = Thrift.readi64(r) + elseif id == Int16(5) && ty == Thrift.BINARY + f_max_value = Thrift.readbinary(r) + elseif id == Int16(6) && ty == Thrift.BINARY + f_min_value = Thrift.readbinary(r) + elseif id == Int16(7) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_is_max_value_exact = ty == Thrift.BOOL_TRUE + elseif id == Int16(8) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_is_min_value_exact = ty == Thrift.BOOL_TRUE + elseif id == Int16(9) && ty == Thrift.I64 + f_nan_count = Thrift.readi64(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return Statistics(f_max, f_min, f_null_count, f_distinct_count, f_max_value, f_min_value, f_is_max_value_exact, f_is_min_value_exact, f_nan_count, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::Statistics) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_max = x.max + if value_max !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BINARY) + Thrift.writebinary!(w, value_max) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_min = x.min + if value_min !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.BINARY) + Thrift.writebinary!(w, value_min) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_null_count = x.null_count + if value_null_count !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I64) + Thrift.writei64!(w, value_null_count) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_distinct_count = x.distinct_count + if value_distinct_count !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.I64) + Thrift.writei64!(w, value_distinct_count) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_max_value = x.max_value + if value_max_value !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.BINARY) + Thrift.writebinary!(w, value_max_value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_min_value = x.min_value + if value_min_value !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.BINARY) + Thrift.writebinary!(w, value_min_value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_is_max_value_exact = x.is_max_value_exact + if value_is_max_value_exact !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), value_is_max_value_exact ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_is_min_value_exact = x.is_min_value_exact + if value_is_min_value_exact !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), value_is_min_value_exact ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_nan_count = x.nan_count + if value_nan_count !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(9), Thrift.I64) + Thrift.writei64!(w, value_nan_count) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct StringType +Base.@kwdef struct StringType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::StringType, b::StringType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::StringType, b::StringType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::StringType, h::UInt) + h = hash(:StringType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{StringType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{StringType}) + return Thrift.decode(r, StringType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::StringType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{StringType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return StringType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::StringType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct UUIDType +Base.@kwdef struct UUIDType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::UUIDType, b::UUIDType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::UUIDType, b::UUIDType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::UUIDType, h::UInt) + h = hash(:UUIDType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{UUIDType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{UUIDType}) + return Thrift.decode(r, UUIDType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::UUIDType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{UUIDType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return UUIDType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::UUIDType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct MapType +Base.@kwdef struct MapType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::MapType, b::MapType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::MapType, b::MapType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::MapType, h::UInt) + h = hash(:MapType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{MapType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{MapType}) + return Thrift.decode(r, MapType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::MapType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{MapType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return MapType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::MapType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct ListType +Base.@kwdef struct ListType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::ListType, b::ListType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::ListType, b::ListType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::ListType, h::UInt) + h = hash(:ListType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{ListType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{ListType}) + return Thrift.decode(r, ListType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::ListType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{ListType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return ListType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::ListType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct EnumType +Base.@kwdef struct EnumType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::EnumType, b::EnumType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::EnumType, b::EnumType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::EnumType, h::UInt) + h = hash(:EnumType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{EnumType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{EnumType}) + return Thrift.decode(r, EnumType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::EnumType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{EnumType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return EnumType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::EnumType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct DateType +Base.@kwdef struct DateType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::DateType, b::DateType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::DateType, b::DateType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::DateType, h::UInt) + h = hash(:DateType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{DateType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{DateType}) + return Thrift.decode(r, DateType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::DateType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{DateType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return DateType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::DateType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct Float16Type +Base.@kwdef struct Float16Type + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::Float16Type, b::Float16Type) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::Float16Type, b::Float16Type) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::Float16Type, h::UInt) + h = hash(:Float16Type, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{Float16Type}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{Float16Type}) + return Thrift.decode(r, Float16Type) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::Float16Type) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{Float16Type}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return Float16Type(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::Float16Type) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct NullType +Base.@kwdef struct NullType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::NullType, b::NullType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::NullType, b::NullType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::NullType, h::UInt) + h = hash(:NullType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{NullType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{NullType}) + return Thrift.decode(r, NullType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::NullType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{NullType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return NullType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::NullType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct DecimalType +Base.@kwdef struct DecimalType + scale::Int32 # 1: required i32 scale + precision::Int32 # 2: required i32 precision + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::DecimalType, b::DecimalType) + return a.scale == b.scale && a.precision == b.precision && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::DecimalType, b::DecimalType) + return isequal(a.scale, b.scale) && isequal(a.precision, b.precision) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::DecimalType, h::UInt) + h = hash(:DecimalType, h) + h = hash(x.scale, h) + h = hash(x.precision, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{DecimalType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{DecimalType}) + return Thrift.decode(r, DecimalType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::DecimalType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{DecimalType}) + Thrift.enter!(r) + f_scale = nothing + f_precision = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_scale = Thrift.readi32(r) + elseif id == Int16(2) && ty == Thrift.I32 + f_precision = Thrift.readi32(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_scale === nothing && Thrift.missingfield(:DecimalType, :scale) + f_precision === nothing && Thrift.missingfield(:DecimalType, :precision) + return DecimalType(f_scale, f_precision, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::DecimalType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.scale) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, x.precision) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct MilliSeconds +Base.@kwdef struct MilliSeconds + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::MilliSeconds, b::MilliSeconds) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::MilliSeconds, b::MilliSeconds) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::MilliSeconds, h::UInt) + h = hash(:MilliSeconds, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{MilliSeconds}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{MilliSeconds}) + return Thrift.decode(r, MilliSeconds) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::MilliSeconds) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{MilliSeconds}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return MilliSeconds(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::MilliSeconds) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct MicroSeconds +Base.@kwdef struct MicroSeconds + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::MicroSeconds, b::MicroSeconds) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::MicroSeconds, b::MicroSeconds) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::MicroSeconds, h::UInt) + h = hash(:MicroSeconds, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{MicroSeconds}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{MicroSeconds}) + return Thrift.decode(r, MicroSeconds) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::MicroSeconds) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{MicroSeconds}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return MicroSeconds(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::MicroSeconds) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct NanoSeconds +Base.@kwdef struct NanoSeconds + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::NanoSeconds, b::NanoSeconds) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::NanoSeconds, b::NanoSeconds) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::NanoSeconds, h::UInt) + h = hash(:NanoSeconds, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{NanoSeconds}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{NanoSeconds}) + return Thrift.decode(r, NanoSeconds) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::NanoSeconds) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{NanoSeconds}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return NanoSeconds(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::NanoSeconds) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union TimeUnit +struct TimeUnit + MILLIS::Union{Nothing, MilliSeconds} # 1: MilliSeconds MILLIS + MICROS::Union{Nothing, MicroSeconds} # 2: MicroSeconds MICROS + NANOS::Union{Nothing, NanoSeconds} # 3: NanoSeconds NANOS + unknown_fields::Vector{Thrift.RawField} + function TimeUnit(MILLIS, MICROS, NANOS, unknown_fields) + Thrift.checkunionargs(:TimeUnit, (MILLIS !== nothing) + (MICROS !== nothing) + (NANOS !== nothing), unknown_fields) + return new(MILLIS, MICROS, NANOS, unknown_fields) + end +end + +function TimeUnit(; MILLIS=nothing, MICROS=nothing, NANOS=nothing, unknown_fields=Thrift.RawField[]) + return TimeUnit(MILLIS, MICROS, NANOS, unknown_fields) +end + +function Base.:(==)(a::TimeUnit, b::TimeUnit) + return a.MILLIS == b.MILLIS && a.MICROS == b.MICROS && a.NANOS == b.NANOS && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::TimeUnit, b::TimeUnit) + return isequal(a.MILLIS, b.MILLIS) && isequal(a.MICROS, b.MICROS) && isequal(a.NANOS, b.NANOS) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::TimeUnit, h::UInt) + h = hash(:TimeUnit, h) + h = hash(x.MILLIS, h) + h = hash(x.MICROS, h) + h = hash(x.NANOS, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{TimeUnit}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{TimeUnit}) + return Thrift.decode(r, TimeUnit) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::TimeUnit) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{TimeUnit}) + Thrift.enter!(r) + f_MILLIS = nothing + f_MICROS = nothing + f_NANOS = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_MILLIS = Thrift.decode(r, MilliSeconds) + elseif id == Int16(2) && ty == Thrift.STRUCT + f_MICROS = Thrift.decode(r, MicroSeconds) + elseif id == Int16(3) && ty == Thrift.STRUCT + f_NANOS = Thrift.decode(r, NanoSeconds) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:TimeUnit, (f_MILLIS !== nothing) + (f_MICROS !== nothing) + (f_NANOS !== nothing), unknown_fields) + return TimeUnit(f_MILLIS, f_MICROS, f_NANOS, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::TimeUnit) + unknown = x.unknown_fields + Thrift.checkunionargs(:TimeUnit, (x.MILLIS !== nothing) + (x.MICROS !== nothing) + (x.NANOS !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_MILLIS = x.MILLIS + if value_MILLIS !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_MILLIS) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_MICROS = x.MICROS + if value_MICROS !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, value_MICROS) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_NANOS = x.NANOS + if value_NANOS !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.STRUCT) + Thrift.encode!(w, value_NANOS) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct TimestampType +Base.@kwdef struct TimestampType + isAdjustedToUTC::Bool # 1: required bool isAdjustedToUTC + unit::TimeUnit # 2: required TimeUnit unit + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::TimestampType, b::TimestampType) + return a.isAdjustedToUTC == b.isAdjustedToUTC && a.unit == b.unit && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::TimestampType, b::TimestampType) + return isequal(a.isAdjustedToUTC, b.isAdjustedToUTC) && isequal(a.unit, b.unit) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::TimestampType, h::UInt) + h = hash(:TimestampType, h) + h = hash(x.isAdjustedToUTC, h) + h = hash(x.unit, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{TimestampType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{TimestampType}) + return Thrift.decode(r, TimestampType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::TimestampType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{TimestampType}) + Thrift.enter!(r) + f_isAdjustedToUTC = nothing + f_unit = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_isAdjustedToUTC = ty == Thrift.BOOL_TRUE + elseif id == Int16(2) && ty == Thrift.STRUCT + f_unit = Thrift.decode(r, TimeUnit) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_isAdjustedToUTC === nothing && Thrift.missingfield(:TimestampType, :isAdjustedToUTC) + f_unit === nothing && Thrift.missingfield(:TimestampType, :unit) + return TimestampType(f_isAdjustedToUTC, f_unit, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::TimestampType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), x.isAdjustedToUTC ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, x.unit) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct TimeType +Base.@kwdef struct TimeType + isAdjustedToUTC::Bool # 1: required bool isAdjustedToUTC + unit::TimeUnit # 2: required TimeUnit unit + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::TimeType, b::TimeType) + return a.isAdjustedToUTC == b.isAdjustedToUTC && a.unit == b.unit && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::TimeType, b::TimeType) + return isequal(a.isAdjustedToUTC, b.isAdjustedToUTC) && isequal(a.unit, b.unit) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::TimeType, h::UInt) + h = hash(:TimeType, h) + h = hash(x.isAdjustedToUTC, h) + h = hash(x.unit, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{TimeType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{TimeType}) + return Thrift.decode(r, TimeType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::TimeType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{TimeType}) + Thrift.enter!(r) + f_isAdjustedToUTC = nothing + f_unit = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_isAdjustedToUTC = ty == Thrift.BOOL_TRUE + elseif id == Int16(2) && ty == Thrift.STRUCT + f_unit = Thrift.decode(r, TimeUnit) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_isAdjustedToUTC === nothing && Thrift.missingfield(:TimeType, :isAdjustedToUTC) + f_unit === nothing && Thrift.missingfield(:TimeType, :unit) + return TimeType(f_isAdjustedToUTC, f_unit, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::TimeType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), x.isAdjustedToUTC ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, x.unit) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct IntType +Base.@kwdef struct IntType + bitWidth::Int8 # 1: required i8 bitWidth + isSigned::Bool # 2: required bool isSigned + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::IntType, b::IntType) + return a.bitWidth == b.bitWidth && a.isSigned == b.isSigned && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::IntType, b::IntType) + return isequal(a.bitWidth, b.bitWidth) && isequal(a.isSigned, b.isSigned) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::IntType, h::UInt) + h = hash(:IntType, h) + h = hash(x.bitWidth, h) + h = hash(x.isSigned, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{IntType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{IntType}) + return Thrift.decode(r, IntType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::IntType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{IntType}) + Thrift.enter!(r) + f_bitWidth = nothing + f_isSigned = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BYTE + f_bitWidth = Thrift.readi8(r) + elseif id == Int16(2) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_isSigned = ty == Thrift.BOOL_TRUE + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_bitWidth === nothing && Thrift.missingfield(:IntType, :bitWidth) + f_isSigned === nothing && Thrift.missingfield(:IntType, :isSigned) + return IntType(f_bitWidth, f_isSigned, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::IntType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BYTE) + Thrift.writei8!(w, x.bitWidth) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), x.isSigned ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct JsonType +Base.@kwdef struct JsonType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::JsonType, b::JsonType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::JsonType, b::JsonType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::JsonType, h::UInt) + h = hash(:JsonType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{JsonType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{JsonType}) + return Thrift.decode(r, JsonType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::JsonType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{JsonType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return JsonType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::JsonType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct BsonType +Base.@kwdef struct BsonType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::BsonType, b::BsonType) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::BsonType, b::BsonType) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::BsonType, h::UInt) + h = hash(:BsonType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{BsonType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{BsonType}) + return Thrift.decode(r, BsonType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::BsonType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{BsonType}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return BsonType(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::BsonType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct VariantType +Base.@kwdef struct VariantType + specification_version::Union{Nothing, Int8} = nothing # 1: optional i8 specification_version + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::VariantType, b::VariantType) + return a.specification_version == b.specification_version && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::VariantType, b::VariantType) + return isequal(a.specification_version, b.specification_version) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::VariantType, h::UInt) + h = hash(:VariantType, h) + h = hash(x.specification_version, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{VariantType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{VariantType}) + return Thrift.decode(r, VariantType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::VariantType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{VariantType}) + Thrift.enter!(r) + f_specification_version = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BYTE + f_specification_version = Thrift.readi8(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return VariantType(f_specification_version, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::VariantType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_specification_version = x.specification_version + if value_specification_version !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BYTE) + Thrift.writei8!(w, value_specification_version) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +module EdgeInterpolationAlgorithm + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const SPHERICAL = T(0) + +const VINCENTY = T(1) + +const THOMAS = T(2) + +const ANDOYER = T(3) + +const KARNEY = T(4) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :SPHERICAL), (Int32(1), :VINCENTY), (Int32(2), :THOMAS), (Int32(3), :ANDOYER), (Int32(4), :KARNEY)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +# Thrift struct GeometryType +Base.@kwdef struct GeometryType + crs::Union{Nothing, String} = nothing # 1: optional string crs + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::GeometryType, b::GeometryType) + return a.crs == b.crs && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::GeometryType, b::GeometryType) + return isequal(a.crs, b.crs) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::GeometryType, h::UInt) + h = hash(:GeometryType, h) + h = hash(x.crs, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{GeometryType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{GeometryType}) + return Thrift.decode(r, GeometryType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::GeometryType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{GeometryType}) + Thrift.enter!(r) + f_crs = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BINARY + f_crs = Thrift.readstring(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return GeometryType(f_crs, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::GeometryType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_crs = x.crs + if value_crs !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BINARY) + Thrift.writestring!(w, value_crs) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct GeographyType +Base.@kwdef struct GeographyType + crs::Union{Nothing, String} = nothing # 1: optional string crs + algorithm::Union{Nothing, EdgeInterpolationAlgorithm.T} = nothing # 2: optional EdgeInterpolationAlgorithm algorithm + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::GeographyType, b::GeographyType) + return a.crs == b.crs && a.algorithm == b.algorithm && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::GeographyType, b::GeographyType) + return isequal(a.crs, b.crs) && isequal(a.algorithm, b.algorithm) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::GeographyType, h::UInt) + h = hash(:GeographyType, h) + h = hash(x.crs, h) + h = hash(x.algorithm, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{GeographyType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{GeographyType}) + return Thrift.decode(r, GeographyType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::GeographyType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{GeographyType}) + Thrift.enter!(r) + f_crs = nothing + f_algorithm = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BINARY + f_crs = Thrift.readstring(r) + elseif id == Int16(2) && ty == Thrift.I32 + f_algorithm = EdgeInterpolationAlgorithm.T(Thrift.readi32(r)) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return GeographyType(f_crs, f_algorithm, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::GeographyType) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_crs = x.crs + if value_crs !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BINARY) + Thrift.writestring!(w, value_crs) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_algorithm = x.algorithm + if value_algorithm !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, value_algorithm.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union LogicalType +struct LogicalType + STRING::Union{Nothing, StringType} # 1: StringType STRING + MAP::Union{Nothing, MapType} # 2: MapType MAP + LIST::Union{Nothing, ListType} # 3: ListType LIST + ENUM::Union{Nothing, EnumType} # 4: EnumType ENUM + DECIMAL::Union{Nothing, DecimalType} # 5: DecimalType DECIMAL + DATE::Union{Nothing, DateType} # 6: DateType DATE + TIME::Union{Nothing, TimeType} # 7: TimeType TIME + TIMESTAMP::Union{Nothing, TimestampType} # 8: TimestampType TIMESTAMP + INTEGER::Union{Nothing, IntType} # 10: IntType INTEGER + UNKNOWN::Union{Nothing, NullType} # 11: NullType UNKNOWN + JSON::Union{Nothing, JsonType} # 12: JsonType JSON + BSON::Union{Nothing, BsonType} # 13: BsonType BSON + UUID::Union{Nothing, UUIDType} # 14: UUIDType UUID + FLOAT16::Union{Nothing, Float16Type} # 15: Float16Type FLOAT16 + VARIANT::Union{Nothing, VariantType} # 16: VariantType VARIANT + GEOMETRY::Union{Nothing, GeometryType} # 17: GeometryType GEOMETRY + GEOGRAPHY::Union{Nothing, GeographyType} # 18: GeographyType GEOGRAPHY + unknown_fields::Vector{Thrift.RawField} + function LogicalType(STRING, MAP, LIST, ENUM, DECIMAL, DATE, TIME, TIMESTAMP, INTEGER, UNKNOWN, JSON, BSON, UUID, FLOAT16, VARIANT, GEOMETRY, GEOGRAPHY, unknown_fields) + Thrift.checkunionargs(:LogicalType, (STRING !== nothing) + (MAP !== nothing) + (LIST !== nothing) + (ENUM !== nothing) + (DECIMAL !== nothing) + (DATE !== nothing) + (TIME !== nothing) + (TIMESTAMP !== nothing) + (INTEGER !== nothing) + (UNKNOWN !== nothing) + (JSON !== nothing) + (BSON !== nothing) + (UUID !== nothing) + (FLOAT16 !== nothing) + (VARIANT !== nothing) + (GEOMETRY !== nothing) + (GEOGRAPHY !== nothing), unknown_fields) + return new(STRING, MAP, LIST, ENUM, DECIMAL, DATE, TIME, TIMESTAMP, INTEGER, UNKNOWN, JSON, BSON, UUID, FLOAT16, VARIANT, GEOMETRY, GEOGRAPHY, unknown_fields) + end +end + +function LogicalType(; STRING=nothing, MAP=nothing, LIST=nothing, ENUM=nothing, DECIMAL=nothing, DATE=nothing, TIME=nothing, TIMESTAMP=nothing, INTEGER=nothing, UNKNOWN=nothing, JSON=nothing, BSON=nothing, UUID=nothing, FLOAT16=nothing, VARIANT=nothing, GEOMETRY=nothing, GEOGRAPHY=nothing, unknown_fields=Thrift.RawField[]) + return LogicalType(STRING, MAP, LIST, ENUM, DECIMAL, DATE, TIME, TIMESTAMP, INTEGER, UNKNOWN, JSON, BSON, UUID, FLOAT16, VARIANT, GEOMETRY, GEOGRAPHY, unknown_fields) +end + +function Base.:(==)(a::LogicalType, b::LogicalType) + return a.STRING == b.STRING && a.MAP == b.MAP && a.LIST == b.LIST && a.ENUM == b.ENUM && a.DECIMAL == b.DECIMAL && a.DATE == b.DATE && a.TIME == b.TIME && a.TIMESTAMP == b.TIMESTAMP && a.INTEGER == b.INTEGER && a.UNKNOWN == b.UNKNOWN && a.JSON == b.JSON && a.BSON == b.BSON && a.UUID == b.UUID && a.FLOAT16 == b.FLOAT16 && a.VARIANT == b.VARIANT && a.GEOMETRY == b.GEOMETRY && a.GEOGRAPHY == b.GEOGRAPHY && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::LogicalType, b::LogicalType) + return isequal(a.STRING, b.STRING) && isequal(a.MAP, b.MAP) && isequal(a.LIST, b.LIST) && isequal(a.ENUM, b.ENUM) && isequal(a.DECIMAL, b.DECIMAL) && isequal(a.DATE, b.DATE) && isequal(a.TIME, b.TIME) && isequal(a.TIMESTAMP, b.TIMESTAMP) && isequal(a.INTEGER, b.INTEGER) && isequal(a.UNKNOWN, b.UNKNOWN) && isequal(a.JSON, b.JSON) && isequal(a.BSON, b.BSON) && isequal(a.UUID, b.UUID) && isequal(a.FLOAT16, b.FLOAT16) && isequal(a.VARIANT, b.VARIANT) && isequal(a.GEOMETRY, b.GEOMETRY) && isequal(a.GEOGRAPHY, b.GEOGRAPHY) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::LogicalType, h::UInt) + h = hash(:LogicalType, h) + h = hash(x.STRING, h) + h = hash(x.MAP, h) + h = hash(x.LIST, h) + h = hash(x.ENUM, h) + h = hash(x.DECIMAL, h) + h = hash(x.DATE, h) + h = hash(x.TIME, h) + h = hash(x.TIMESTAMP, h) + h = hash(x.INTEGER, h) + h = hash(x.UNKNOWN, h) + h = hash(x.JSON, h) + h = hash(x.BSON, h) + h = hash(x.UUID, h) + h = hash(x.FLOAT16, h) + h = hash(x.VARIANT, h) + h = hash(x.GEOMETRY, h) + h = hash(x.GEOGRAPHY, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{LogicalType}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{LogicalType}) + return Thrift.decode(r, LogicalType) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::LogicalType) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{LogicalType}) + Thrift.enter!(r) + f_STRING = nothing + f_MAP = nothing + f_LIST = nothing + f_ENUM = nothing + f_DECIMAL = nothing + f_DATE = nothing + f_TIME = nothing + f_TIMESTAMP = nothing + f_INTEGER = nothing + f_UNKNOWN = nothing + f_JSON = nothing + f_BSON = nothing + f_UUID = nothing + f_FLOAT16 = nothing + f_VARIANT = nothing + f_GEOMETRY = nothing + f_GEOGRAPHY = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_STRING = Thrift.decode(r, StringType) + elseif id == Int16(2) && ty == Thrift.STRUCT + f_MAP = Thrift.decode(r, MapType) + elseif id == Int16(3) && ty == Thrift.STRUCT + f_LIST = Thrift.decode(r, ListType) + elseif id == Int16(4) && ty == Thrift.STRUCT + f_ENUM = Thrift.decode(r, EnumType) + elseif id == Int16(5) && ty == Thrift.STRUCT + f_DECIMAL = Thrift.decode(r, DecimalType) + elseif id == Int16(6) && ty == Thrift.STRUCT + f_DATE = Thrift.decode(r, DateType) + elseif id == Int16(7) && ty == Thrift.STRUCT + f_TIME = Thrift.decode(r, TimeType) + elseif id == Int16(8) && ty == Thrift.STRUCT + f_TIMESTAMP = Thrift.decode(r, TimestampType) + elseif id == Int16(10) && ty == Thrift.STRUCT + f_INTEGER = Thrift.decode(r, IntType) + elseif id == Int16(11) && ty == Thrift.STRUCT + f_UNKNOWN = Thrift.decode(r, NullType) + elseif id == Int16(12) && ty == Thrift.STRUCT + f_JSON = Thrift.decode(r, JsonType) + elseif id == Int16(13) && ty == Thrift.STRUCT + f_BSON = Thrift.decode(r, BsonType) + elseif id == Int16(14) && ty == Thrift.STRUCT + f_UUID = Thrift.decode(r, UUIDType) + elseif id == Int16(15) && ty == Thrift.STRUCT + f_FLOAT16 = Thrift.decode(r, Float16Type) + elseif id == Int16(16) && ty == Thrift.STRUCT + f_VARIANT = Thrift.decode(r, VariantType) + elseif id == Int16(17) && ty == Thrift.STRUCT + f_GEOMETRY = Thrift.decode(r, GeometryType) + elseif id == Int16(18) && ty == Thrift.STRUCT + f_GEOGRAPHY = Thrift.decode(r, GeographyType) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:LogicalType, (f_STRING !== nothing) + (f_MAP !== nothing) + (f_LIST !== nothing) + (f_ENUM !== nothing) + (f_DECIMAL !== nothing) + (f_DATE !== nothing) + (f_TIME !== nothing) + (f_TIMESTAMP !== nothing) + (f_INTEGER !== nothing) + (f_UNKNOWN !== nothing) + (f_JSON !== nothing) + (f_BSON !== nothing) + (f_UUID !== nothing) + (f_FLOAT16 !== nothing) + (f_VARIANT !== nothing) + (f_GEOMETRY !== nothing) + (f_GEOGRAPHY !== nothing), unknown_fields) + return LogicalType(f_STRING, f_MAP, f_LIST, f_ENUM, f_DECIMAL, f_DATE, f_TIME, f_TIMESTAMP, f_INTEGER, f_UNKNOWN, f_JSON, f_BSON, f_UUID, f_FLOAT16, f_VARIANT, f_GEOMETRY, f_GEOGRAPHY, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::LogicalType) + unknown = x.unknown_fields + Thrift.checkunionargs(:LogicalType, (x.STRING !== nothing) + (x.MAP !== nothing) + (x.LIST !== nothing) + (x.ENUM !== nothing) + (x.DECIMAL !== nothing) + (x.DATE !== nothing) + (x.TIME !== nothing) + (x.TIMESTAMP !== nothing) + (x.INTEGER !== nothing) + (x.UNKNOWN !== nothing) + (x.JSON !== nothing) + (x.BSON !== nothing) + (x.UUID !== nothing) + (x.FLOAT16 !== nothing) + (x.VARIANT !== nothing) + (x.GEOMETRY !== nothing) + (x.GEOGRAPHY !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_STRING = x.STRING + if value_STRING !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_STRING) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_MAP = x.MAP + if value_MAP !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, value_MAP) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_LIST = x.LIST + if value_LIST !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.STRUCT) + Thrift.encode!(w, value_LIST) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_ENUM = x.ENUM + if value_ENUM !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.STRUCT) + Thrift.encode!(w, value_ENUM) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_DECIMAL = x.DECIMAL + if value_DECIMAL !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.STRUCT) + Thrift.encode!(w, value_DECIMAL) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_DATE = x.DATE + if value_DATE !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.STRUCT) + Thrift.encode!(w, value_DATE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_TIME = x.TIME + if value_TIME !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.STRUCT) + Thrift.encode!(w, value_TIME) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_TIMESTAMP = x.TIMESTAMP + if value_TIMESTAMP !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.STRUCT) + Thrift.encode!(w, value_TIMESTAMP) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_INTEGER = x.INTEGER + if value_INTEGER !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(10), Thrift.STRUCT) + Thrift.encode!(w, value_INTEGER) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_UNKNOWN = x.UNKNOWN + if value_UNKNOWN !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(11), Thrift.STRUCT) + Thrift.encode!(w, value_UNKNOWN) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_JSON = x.JSON + if value_JSON !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(12), Thrift.STRUCT) + Thrift.encode!(w, value_JSON) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_BSON = x.BSON + if value_BSON !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(13), Thrift.STRUCT) + Thrift.encode!(w, value_BSON) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_UUID = x.UUID + if value_UUID !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(14), Thrift.STRUCT) + Thrift.encode!(w, value_UUID) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_FLOAT16 = x.FLOAT16 + if value_FLOAT16 !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(15), Thrift.STRUCT) + Thrift.encode!(w, value_FLOAT16) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_VARIANT = x.VARIANT + if value_VARIANT !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(16), Thrift.STRUCT) + Thrift.encode!(w, value_VARIANT) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_GEOMETRY = x.GEOMETRY + if value_GEOMETRY !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(17), Thrift.STRUCT) + Thrift.encode!(w, value_GEOMETRY) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_GEOGRAPHY = x.GEOGRAPHY + if value_GEOGRAPHY !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(18), Thrift.STRUCT) + Thrift.encode!(w, value_GEOGRAPHY) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct SchemaElement +Base.@kwdef struct SchemaElement + type_::Union{Nothing, Type.T} = nothing # 1: optional Type type + type_length::Union{Nothing, Int32} = nothing # 2: optional i32 type_length + repetition_type::Union{Nothing, FieldRepetitionType.T} = nothing # 3: optional FieldRepetitionType repetition_type + name::String # 4: required string name + num_children::Union{Nothing, Int32} = nothing # 5: optional i32 num_children + converted_type::Union{Nothing, ConvertedType.T} = nothing # 6: optional ConvertedType converted_type + scale::Union{Nothing, Int32} = nothing # 7: optional i32 scale + precision::Union{Nothing, Int32} = nothing # 8: optional i32 precision + field_id::Union{Nothing, Int32} = nothing # 9: optional i32 field_id + logicalType::Union{Nothing, LogicalType} = nothing # 10: optional LogicalType logicalType + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::SchemaElement, b::SchemaElement) + return a.type_ == b.type_ && a.type_length == b.type_length && a.repetition_type == b.repetition_type && a.name == b.name && a.num_children == b.num_children && a.converted_type == b.converted_type && a.scale == b.scale && a.precision == b.precision && a.field_id == b.field_id && a.logicalType == b.logicalType && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::SchemaElement, b::SchemaElement) + return isequal(a.type_, b.type_) && isequal(a.type_length, b.type_length) && isequal(a.repetition_type, b.repetition_type) && isequal(a.name, b.name) && isequal(a.num_children, b.num_children) && isequal(a.converted_type, b.converted_type) && isequal(a.scale, b.scale) && isequal(a.precision, b.precision) && isequal(a.field_id, b.field_id) && isequal(a.logicalType, b.logicalType) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::SchemaElement, h::UInt) + h = hash(:SchemaElement, h) + h = hash(x.type_, h) + h = hash(x.type_length, h) + h = hash(x.repetition_type, h) + h = hash(x.name, h) + h = hash(x.num_children, h) + h = hash(x.converted_type, h) + h = hash(x.scale, h) + h = hash(x.precision, h) + h = hash(x.field_id, h) + h = hash(x.logicalType, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{SchemaElement}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{SchemaElement}) + return Thrift.decode(r, SchemaElement) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::SchemaElement) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{SchemaElement}) + Thrift.enter!(r) + f_type_ = nothing + f_type_length = nothing + f_repetition_type = nothing + f_name = nothing + f_num_children = nothing + f_converted_type = nothing + f_scale = nothing + f_precision = nothing + f_field_id = nothing + f_logicalType = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_type_ = Type.T(Thrift.readi32(r)) + elseif id == Int16(2) && ty == Thrift.I32 + f_type_length = Thrift.readi32(r) + elseif id == Int16(3) && ty == Thrift.I32 + f_repetition_type = FieldRepetitionType.T(Thrift.readi32(r)) + elseif id == Int16(4) && ty == Thrift.BINARY + f_name = Thrift.readstring(r) + elseif id == Int16(5) && ty == Thrift.I32 + f_num_children = Thrift.readi32(r) + elseif id == Int16(6) && ty == Thrift.I32 + f_converted_type = ConvertedType.T(Thrift.readi32(r)) + elseif id == Int16(7) && ty == Thrift.I32 + f_scale = Thrift.readi32(r) + elseif id == Int16(8) && ty == Thrift.I32 + f_precision = Thrift.readi32(r) + elseif id == Int16(9) && ty == Thrift.I32 + f_field_id = Thrift.readi32(r) + elseif id == Int16(10) && ty == Thrift.STRUCT + f_logicalType = Thrift.decode(r, LogicalType) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_name === nothing && Thrift.missingfield(:SchemaElement, :name) + return SchemaElement(f_type_, f_type_length, f_repetition_type, f_name, f_num_children, f_converted_type, f_scale, f_precision, f_field_id, f_logicalType, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::SchemaElement) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_type_ = x.type_ + if value_type_ !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, value_type_.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_type_length = x.type_length + if value_type_length !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, value_type_length) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_repetition_type = x.repetition_type + if value_repetition_type !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I32) + Thrift.writei32!(w, value_repetition_type.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.BINARY) + Thrift.writestring!(w, x.name) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_num_children = x.num_children + if value_num_children !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.I32) + Thrift.writei32!(w, value_num_children) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_converted_type = x.converted_type + if value_converted_type !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.I32) + Thrift.writei32!(w, value_converted_type.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_scale = x.scale + if value_scale !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.I32) + Thrift.writei32!(w, value_scale) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_precision = x.precision + if value_precision !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.I32) + Thrift.writei32!(w, value_precision) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_field_id = x.field_id + if value_field_id !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(9), Thrift.I32) + Thrift.writei32!(w, value_field_id) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_logicalType = x.logicalType + if value_logicalType !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(10), Thrift.STRUCT) + Thrift.encode!(w, value_logicalType) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +module Encoding + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const PLAIN = T(0) + +const PLAIN_DICTIONARY = T(2) + +const RLE = T(3) + +const BIT_PACKED = T(4) + +const DELTA_BINARY_PACKED = T(5) + +const DELTA_LENGTH_BYTE_ARRAY = T(6) + +const DELTA_BYTE_ARRAY = T(7) + +const RLE_DICTIONARY = T(8) + +const BYTE_STREAM_SPLIT = T(9) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :PLAIN), (Int32(2), :PLAIN_DICTIONARY), (Int32(3), :RLE), (Int32(4), :BIT_PACKED), (Int32(5), :DELTA_BINARY_PACKED), (Int32(6), :DELTA_LENGTH_BYTE_ARRAY), (Int32(7), :DELTA_BYTE_ARRAY), (Int32(8), :RLE_DICTIONARY), (Int32(9), :BYTE_STREAM_SPLIT)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +module CompressionCodec + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const UNCOMPRESSED = T(0) + +const SNAPPY = T(1) + +const GZIP = T(2) + +const LZO = T(3) + +const BROTLI = T(4) + +const LZ4 = T(5) + +const ZSTD = T(6) + +const LZ4_RAW = T(7) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :UNCOMPRESSED), (Int32(1), :SNAPPY), (Int32(2), :GZIP), (Int32(3), :LZO), (Int32(4), :BROTLI), (Int32(5), :LZ4), (Int32(6), :ZSTD), (Int32(7), :LZ4_RAW)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +module PageType + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const DATA_PAGE = T(0) + +const INDEX_PAGE = T(1) + +const DICTIONARY_PAGE = T(2) + +const DATA_PAGE_V2 = T(3) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :DATA_PAGE), (Int32(1), :INDEX_PAGE), (Int32(2), :DICTIONARY_PAGE), (Int32(3), :DATA_PAGE_V2)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +module BoundaryOrder + +import ..Thrift + +struct T <: Thrift.ThriftEnum + value::Int32 +end + +const UNORDERED = T(0) + +const ASCENDING = T(1) + +const DESCENDING = T(2) + +function Thrift.enumnames(::Core.Type{T}) + return ((Int32(0), :UNORDERED), (Int32(1), :ASCENDING), (Int32(2), :DESCENDING)) +end + +function Thrift.typecode(::Core.Type{T}) + return Thrift.I32 +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T}) + return T(Thrift.readi32(r)) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::T) + Thrift.writei32!(w, x.value) + return +end + +end + +# Thrift struct DataPageHeader +Base.@kwdef struct DataPageHeader + num_values::Int32 # 1: required i32 num_values + encoding::Encoding.T # 2: required Encoding encoding + definition_level_encoding::Encoding.T # 3: required Encoding definition_level_encoding + repetition_level_encoding::Encoding.T # 4: required Encoding repetition_level_encoding + statistics::Union{Nothing, Statistics} = nothing # 5: optional Statistics statistics + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::DataPageHeader, b::DataPageHeader) + return a.num_values == b.num_values && a.encoding == b.encoding && a.definition_level_encoding == b.definition_level_encoding && a.repetition_level_encoding == b.repetition_level_encoding && a.statistics == b.statistics && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::DataPageHeader, b::DataPageHeader) + return isequal(a.num_values, b.num_values) && isequal(a.encoding, b.encoding) && isequal(a.definition_level_encoding, b.definition_level_encoding) && isequal(a.repetition_level_encoding, b.repetition_level_encoding) && isequal(a.statistics, b.statistics) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::DataPageHeader, h::UInt) + h = hash(:DataPageHeader, h) + h = hash(x.num_values, h) + h = hash(x.encoding, h) + h = hash(x.definition_level_encoding, h) + h = hash(x.repetition_level_encoding, h) + h = hash(x.statistics, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{DataPageHeader}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{DataPageHeader}) + return Thrift.decode(r, DataPageHeader) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::DataPageHeader) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{DataPageHeader}) + Thrift.enter!(r) + f_num_values = nothing + f_encoding = nothing + f_definition_level_encoding = nothing + f_repetition_level_encoding = nothing + f_statistics = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_num_values = Thrift.readi32(r) + elseif id == Int16(2) && ty == Thrift.I32 + f_encoding = Encoding.T(Thrift.readi32(r)) + elseif id == Int16(3) && ty == Thrift.I32 + f_definition_level_encoding = Encoding.T(Thrift.readi32(r)) + elseif id == Int16(4) && ty == Thrift.I32 + f_repetition_level_encoding = Encoding.T(Thrift.readi32(r)) + elseif id == Int16(5) && ty == Thrift.STRUCT + f_statistics = Thrift.decode(r, Statistics) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_num_values === nothing && Thrift.missingfield(:DataPageHeader, :num_values) + f_encoding === nothing && Thrift.missingfield(:DataPageHeader, :encoding) + f_definition_level_encoding === nothing && Thrift.missingfield(:DataPageHeader, :definition_level_encoding) + f_repetition_level_encoding === nothing && Thrift.missingfield(:DataPageHeader, :repetition_level_encoding) + return DataPageHeader(f_num_values, f_encoding, f_definition_level_encoding, f_repetition_level_encoding, f_statistics, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::DataPageHeader) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.num_values) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, x.encoding.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I32) + Thrift.writei32!(w, x.definition_level_encoding.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.I32) + Thrift.writei32!(w, x.repetition_level_encoding.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_statistics = x.statistics + if value_statistics !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.STRUCT) + Thrift.encode!(w, value_statistics) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct IndexPageHeader +Base.@kwdef struct IndexPageHeader + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::IndexPageHeader, b::IndexPageHeader) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::IndexPageHeader, b::IndexPageHeader) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::IndexPageHeader, h::UInt) + h = hash(:IndexPageHeader, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{IndexPageHeader}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{IndexPageHeader}) + return Thrift.decode(r, IndexPageHeader) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::IndexPageHeader) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{IndexPageHeader}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return IndexPageHeader(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::IndexPageHeader) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct DictionaryPageHeader +Base.@kwdef struct DictionaryPageHeader + num_values::Int32 # 1: required i32 num_values + encoding::Encoding.T # 2: required Encoding encoding + is_sorted::Union{Nothing, Bool} = nothing # 3: optional bool is_sorted + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::DictionaryPageHeader, b::DictionaryPageHeader) + return a.num_values == b.num_values && a.encoding == b.encoding && a.is_sorted == b.is_sorted && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::DictionaryPageHeader, b::DictionaryPageHeader) + return isequal(a.num_values, b.num_values) && isequal(a.encoding, b.encoding) && isequal(a.is_sorted, b.is_sorted) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::DictionaryPageHeader, h::UInt) + h = hash(:DictionaryPageHeader, h) + h = hash(x.num_values, h) + h = hash(x.encoding, h) + h = hash(x.is_sorted, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{DictionaryPageHeader}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{DictionaryPageHeader}) + return Thrift.decode(r, DictionaryPageHeader) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::DictionaryPageHeader) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{DictionaryPageHeader}) + Thrift.enter!(r) + f_num_values = nothing + f_encoding = nothing + f_is_sorted = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_num_values = Thrift.readi32(r) + elseif id == Int16(2) && ty == Thrift.I32 + f_encoding = Encoding.T(Thrift.readi32(r)) + elseif id == Int16(3) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_is_sorted = ty == Thrift.BOOL_TRUE + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_num_values === nothing && Thrift.missingfield(:DictionaryPageHeader, :num_values) + f_encoding === nothing && Thrift.missingfield(:DictionaryPageHeader, :encoding) + return DictionaryPageHeader(f_num_values, f_encoding, f_is_sorted, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::DictionaryPageHeader) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.num_values) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, x.encoding.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_is_sorted = x.is_sorted + if value_is_sorted !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), value_is_sorted ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct DataPageHeaderV2 +Base.@kwdef struct DataPageHeaderV2 + num_values::Int32 # 1: required i32 num_values + num_nulls::Int32 # 2: required i32 num_nulls + num_rows::Int32 # 3: required i32 num_rows + encoding::Encoding.T # 4: required Encoding encoding + definition_levels_byte_length::Int32 # 5: required i32 definition_levels_byte_length + repetition_levels_byte_length::Int32 # 6: required i32 repetition_levels_byte_length + is_compressed::Union{Nothing, Bool} = nothing # 7: optional bool is_compressed = true + statistics::Union{Nothing, Statistics} = nothing # 8: optional Statistics statistics + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::DataPageHeaderV2, b::DataPageHeaderV2) + return a.num_values == b.num_values && a.num_nulls == b.num_nulls && a.num_rows == b.num_rows && a.encoding == b.encoding && a.definition_levels_byte_length == b.definition_levels_byte_length && a.repetition_levels_byte_length == b.repetition_levels_byte_length && a.is_compressed == b.is_compressed && a.statistics == b.statistics && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::DataPageHeaderV2, b::DataPageHeaderV2) + return isequal(a.num_values, b.num_values) && isequal(a.num_nulls, b.num_nulls) && isequal(a.num_rows, b.num_rows) && isequal(a.encoding, b.encoding) && isequal(a.definition_levels_byte_length, b.definition_levels_byte_length) && isequal(a.repetition_levels_byte_length, b.repetition_levels_byte_length) && isequal(a.is_compressed, b.is_compressed) && isequal(a.statistics, b.statistics) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::DataPageHeaderV2, h::UInt) + h = hash(:DataPageHeaderV2, h) + h = hash(x.num_values, h) + h = hash(x.num_nulls, h) + h = hash(x.num_rows, h) + h = hash(x.encoding, h) + h = hash(x.definition_levels_byte_length, h) + h = hash(x.repetition_levels_byte_length, h) + h = hash(x.is_compressed, h) + h = hash(x.statistics, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{DataPageHeaderV2}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{DataPageHeaderV2}) + return Thrift.decode(r, DataPageHeaderV2) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::DataPageHeaderV2) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{DataPageHeaderV2}) + Thrift.enter!(r) + f_num_values = nothing + f_num_nulls = nothing + f_num_rows = nothing + f_encoding = nothing + f_definition_levels_byte_length = nothing + f_repetition_levels_byte_length = nothing + f_is_compressed = nothing + f_statistics = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_num_values = Thrift.readi32(r) + elseif id == Int16(2) && ty == Thrift.I32 + f_num_nulls = Thrift.readi32(r) + elseif id == Int16(3) && ty == Thrift.I32 + f_num_rows = Thrift.readi32(r) + elseif id == Int16(4) && ty == Thrift.I32 + f_encoding = Encoding.T(Thrift.readi32(r)) + elseif id == Int16(5) && ty == Thrift.I32 + f_definition_levels_byte_length = Thrift.readi32(r) + elseif id == Int16(6) && ty == Thrift.I32 + f_repetition_levels_byte_length = Thrift.readi32(r) + elseif id == Int16(7) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_is_compressed = ty == Thrift.BOOL_TRUE + elseif id == Int16(8) && ty == Thrift.STRUCT + f_statistics = Thrift.decode(r, Statistics) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_num_values === nothing && Thrift.missingfield(:DataPageHeaderV2, :num_values) + f_num_nulls === nothing && Thrift.missingfield(:DataPageHeaderV2, :num_nulls) + f_num_rows === nothing && Thrift.missingfield(:DataPageHeaderV2, :num_rows) + f_encoding === nothing && Thrift.missingfield(:DataPageHeaderV2, :encoding) + f_definition_levels_byte_length === nothing && Thrift.missingfield(:DataPageHeaderV2, :definition_levels_byte_length) + f_repetition_levels_byte_length === nothing && Thrift.missingfield(:DataPageHeaderV2, :repetition_levels_byte_length) + return DataPageHeaderV2(f_num_values, f_num_nulls, f_num_rows, f_encoding, f_definition_levels_byte_length, f_repetition_levels_byte_length, f_is_compressed, f_statistics, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::DataPageHeaderV2) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.num_values) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, x.num_nulls) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I32) + Thrift.writei32!(w, x.num_rows) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.I32) + Thrift.writei32!(w, x.encoding.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.I32) + Thrift.writei32!(w, x.definition_levels_byte_length) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.I32) + Thrift.writei32!(w, x.repetition_levels_byte_length) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_is_compressed = x.is_compressed + if value_is_compressed !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), value_is_compressed ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_statistics = x.statistics + if value_statistics !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.STRUCT) + Thrift.encode!(w, value_statistics) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct SplitBlockAlgorithm +Base.@kwdef struct SplitBlockAlgorithm + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::SplitBlockAlgorithm, b::SplitBlockAlgorithm) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::SplitBlockAlgorithm, b::SplitBlockAlgorithm) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::SplitBlockAlgorithm, h::UInt) + h = hash(:SplitBlockAlgorithm, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{SplitBlockAlgorithm}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{SplitBlockAlgorithm}) + return Thrift.decode(r, SplitBlockAlgorithm) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::SplitBlockAlgorithm) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{SplitBlockAlgorithm}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return SplitBlockAlgorithm(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::SplitBlockAlgorithm) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union BloomFilterAlgorithm +struct BloomFilterAlgorithm + BLOCK::Union{Nothing, SplitBlockAlgorithm} # 1: SplitBlockAlgorithm BLOCK + unknown_fields::Vector{Thrift.RawField} + function BloomFilterAlgorithm(BLOCK, unknown_fields) + Thrift.checkunionargs(:BloomFilterAlgorithm, (BLOCK !== nothing), unknown_fields) + return new(BLOCK, unknown_fields) + end +end + +function BloomFilterAlgorithm(; BLOCK=nothing, unknown_fields=Thrift.RawField[]) + return BloomFilterAlgorithm(BLOCK, unknown_fields) +end + +function Base.:(==)(a::BloomFilterAlgorithm, b::BloomFilterAlgorithm) + return a.BLOCK == b.BLOCK && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::BloomFilterAlgorithm, b::BloomFilterAlgorithm) + return isequal(a.BLOCK, b.BLOCK) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::BloomFilterAlgorithm, h::UInt) + h = hash(:BloomFilterAlgorithm, h) + h = hash(x.BLOCK, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{BloomFilterAlgorithm}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{BloomFilterAlgorithm}) + return Thrift.decode(r, BloomFilterAlgorithm) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::BloomFilterAlgorithm) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{BloomFilterAlgorithm}) + Thrift.enter!(r) + f_BLOCK = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_BLOCK = Thrift.decode(r, SplitBlockAlgorithm) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:BloomFilterAlgorithm, (f_BLOCK !== nothing), unknown_fields) + return BloomFilterAlgorithm(f_BLOCK, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::BloomFilterAlgorithm) + unknown = x.unknown_fields + Thrift.checkunionargs(:BloomFilterAlgorithm, (x.BLOCK !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_BLOCK = x.BLOCK + if value_BLOCK !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_BLOCK) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct XxHash +Base.@kwdef struct XxHash + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::XxHash, b::XxHash) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::XxHash, b::XxHash) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::XxHash, h::UInt) + h = hash(:XxHash, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{XxHash}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{XxHash}) + return Thrift.decode(r, XxHash) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::XxHash) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{XxHash}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return XxHash(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::XxHash) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union BloomFilterHash +struct BloomFilterHash + XXHASH::Union{Nothing, XxHash} # 1: XxHash XXHASH + unknown_fields::Vector{Thrift.RawField} + function BloomFilterHash(XXHASH, unknown_fields) + Thrift.checkunionargs(:BloomFilterHash, (XXHASH !== nothing), unknown_fields) + return new(XXHASH, unknown_fields) + end +end + +function BloomFilterHash(; XXHASH=nothing, unknown_fields=Thrift.RawField[]) + return BloomFilterHash(XXHASH, unknown_fields) +end + +function Base.:(==)(a::BloomFilterHash, b::BloomFilterHash) + return a.XXHASH == b.XXHASH && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::BloomFilterHash, b::BloomFilterHash) + return isequal(a.XXHASH, b.XXHASH) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::BloomFilterHash, h::UInt) + h = hash(:BloomFilterHash, h) + h = hash(x.XXHASH, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{BloomFilterHash}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{BloomFilterHash}) + return Thrift.decode(r, BloomFilterHash) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::BloomFilterHash) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{BloomFilterHash}) + Thrift.enter!(r) + f_XXHASH = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_XXHASH = Thrift.decode(r, XxHash) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:BloomFilterHash, (f_XXHASH !== nothing), unknown_fields) + return BloomFilterHash(f_XXHASH, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::BloomFilterHash) + unknown = x.unknown_fields + Thrift.checkunionargs(:BloomFilterHash, (x.XXHASH !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_XXHASH = x.XXHASH + if value_XXHASH !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_XXHASH) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct Uncompressed +Base.@kwdef struct Uncompressed + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::Uncompressed, b::Uncompressed) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::Uncompressed, b::Uncompressed) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::Uncompressed, h::UInt) + h = hash(:Uncompressed, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{Uncompressed}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{Uncompressed}) + return Thrift.decode(r, Uncompressed) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::Uncompressed) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{Uncompressed}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return Uncompressed(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::Uncompressed) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union BloomFilterCompression +struct BloomFilterCompression + UNCOMPRESSED::Union{Nothing, Uncompressed} # 1: Uncompressed UNCOMPRESSED + unknown_fields::Vector{Thrift.RawField} + function BloomFilterCompression(UNCOMPRESSED, unknown_fields) + Thrift.checkunionargs(:BloomFilterCompression, (UNCOMPRESSED !== nothing), unknown_fields) + return new(UNCOMPRESSED, unknown_fields) + end +end + +function BloomFilterCompression(; UNCOMPRESSED=nothing, unknown_fields=Thrift.RawField[]) + return BloomFilterCompression(UNCOMPRESSED, unknown_fields) +end + +function Base.:(==)(a::BloomFilterCompression, b::BloomFilterCompression) + return a.UNCOMPRESSED == b.UNCOMPRESSED && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::BloomFilterCompression, b::BloomFilterCompression) + return isequal(a.UNCOMPRESSED, b.UNCOMPRESSED) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::BloomFilterCompression, h::UInt) + h = hash(:BloomFilterCompression, h) + h = hash(x.UNCOMPRESSED, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{BloomFilterCompression}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{BloomFilterCompression}) + return Thrift.decode(r, BloomFilterCompression) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::BloomFilterCompression) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{BloomFilterCompression}) + Thrift.enter!(r) + f_UNCOMPRESSED = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_UNCOMPRESSED = Thrift.decode(r, Uncompressed) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:BloomFilterCompression, (f_UNCOMPRESSED !== nothing), unknown_fields) + return BloomFilterCompression(f_UNCOMPRESSED, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::BloomFilterCompression) + unknown = x.unknown_fields + Thrift.checkunionargs(:BloomFilterCompression, (x.UNCOMPRESSED !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_UNCOMPRESSED = x.UNCOMPRESSED + if value_UNCOMPRESSED !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_UNCOMPRESSED) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct BloomFilterHeader +Base.@kwdef struct BloomFilterHeader + numBytes::Int32 # 1: required i32 numBytes + algorithm::BloomFilterAlgorithm # 2: required BloomFilterAlgorithm algorithm + hash::BloomFilterHash # 3: required BloomFilterHash hash + compression::BloomFilterCompression # 4: required BloomFilterCompression compression + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::BloomFilterHeader, b::BloomFilterHeader) + return a.numBytes == b.numBytes && a.algorithm == b.algorithm && a.hash == b.hash && a.compression == b.compression && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::BloomFilterHeader, b::BloomFilterHeader) + return isequal(a.numBytes, b.numBytes) && isequal(a.algorithm, b.algorithm) && isequal(a.hash, b.hash) && isequal(a.compression, b.compression) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::BloomFilterHeader, h::UInt) + h = hash(:BloomFilterHeader, h) + h = hash(x.numBytes, h) + h = hash(x.algorithm, h) + h = hash(x.hash, h) + h = hash(x.compression, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{BloomFilterHeader}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{BloomFilterHeader}) + return Thrift.decode(r, BloomFilterHeader) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::BloomFilterHeader) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{BloomFilterHeader}) + Thrift.enter!(r) + f_numBytes = nothing + f_algorithm = nothing + f_hash = nothing + f_compression = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_numBytes = Thrift.readi32(r) + elseif id == Int16(2) && ty == Thrift.STRUCT + f_algorithm = Thrift.decode(r, BloomFilterAlgorithm) + elseif id == Int16(3) && ty == Thrift.STRUCT + f_hash = Thrift.decode(r, BloomFilterHash) + elseif id == Int16(4) && ty == Thrift.STRUCT + f_compression = Thrift.decode(r, BloomFilterCompression) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_numBytes === nothing && Thrift.missingfield(:BloomFilterHeader, :numBytes) + f_algorithm === nothing && Thrift.missingfield(:BloomFilterHeader, :algorithm) + f_hash === nothing && Thrift.missingfield(:BloomFilterHeader, :hash) + f_compression === nothing && Thrift.missingfield(:BloomFilterHeader, :compression) + return BloomFilterHeader(f_numBytes, f_algorithm, f_hash, f_compression, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::BloomFilterHeader) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.numBytes) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, x.algorithm) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.STRUCT) + Thrift.encode!(w, x.hash) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.STRUCT) + Thrift.encode!(w, x.compression) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct PageHeader +Base.@kwdef struct PageHeader + type_::PageType.T # 1: required PageType type + uncompressed_page_size::Int32 # 2: required i32 uncompressed_page_size + compressed_page_size::Int32 # 3: required i32 compressed_page_size + crc::Union{Nothing, Int32} = nothing # 4: optional i32 crc + data_page_header::Union{Nothing, DataPageHeader} = nothing # 5: optional DataPageHeader data_page_header + index_page_header::Union{Nothing, IndexPageHeader} = nothing # 6: optional IndexPageHeader index_page_header + dictionary_page_header::Union{Nothing, DictionaryPageHeader} = nothing # 7: optional DictionaryPageHeader dictionary_page_header + data_page_header_v2::Union{Nothing, DataPageHeaderV2} = nothing # 8: optional DataPageHeaderV2 data_page_header_v2 + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::PageHeader, b::PageHeader) + return a.type_ == b.type_ && a.uncompressed_page_size == b.uncompressed_page_size && a.compressed_page_size == b.compressed_page_size && a.crc == b.crc && a.data_page_header == b.data_page_header && a.index_page_header == b.index_page_header && a.dictionary_page_header == b.dictionary_page_header && a.data_page_header_v2 == b.data_page_header_v2 && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::PageHeader, b::PageHeader) + return isequal(a.type_, b.type_) && isequal(a.uncompressed_page_size, b.uncompressed_page_size) && isequal(a.compressed_page_size, b.compressed_page_size) && isequal(a.crc, b.crc) && isequal(a.data_page_header, b.data_page_header) && isequal(a.index_page_header, b.index_page_header) && isequal(a.dictionary_page_header, b.dictionary_page_header) && isequal(a.data_page_header_v2, b.data_page_header_v2) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::PageHeader, h::UInt) + h = hash(:PageHeader, h) + h = hash(x.type_, h) + h = hash(x.uncompressed_page_size, h) + h = hash(x.compressed_page_size, h) + h = hash(x.crc, h) + h = hash(x.data_page_header, h) + h = hash(x.index_page_header, h) + h = hash(x.dictionary_page_header, h) + h = hash(x.data_page_header_v2, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{PageHeader}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{PageHeader}) + return Thrift.decode(r, PageHeader) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::PageHeader) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{PageHeader}) + Thrift.enter!(r) + f_type_ = nothing + f_uncompressed_page_size = nothing + f_compressed_page_size = nothing + f_crc = nothing + f_data_page_header = nothing + f_index_page_header = nothing + f_dictionary_page_header = nothing + f_data_page_header_v2 = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_type_ = PageType.T(Thrift.readi32(r)) + elseif id == Int16(2) && ty == Thrift.I32 + f_uncompressed_page_size = Thrift.readi32(r) + elseif id == Int16(3) && ty == Thrift.I32 + f_compressed_page_size = Thrift.readi32(r) + elseif id == Int16(4) && ty == Thrift.I32 + f_crc = Thrift.readi32(r) + elseif id == Int16(5) && ty == Thrift.STRUCT + f_data_page_header = Thrift.decode(r, DataPageHeader) + elseif id == Int16(6) && ty == Thrift.STRUCT + f_index_page_header = Thrift.decode(r, IndexPageHeader) + elseif id == Int16(7) && ty == Thrift.STRUCT + f_dictionary_page_header = Thrift.decode(r, DictionaryPageHeader) + elseif id == Int16(8) && ty == Thrift.STRUCT + f_data_page_header_v2 = Thrift.decode(r, DataPageHeaderV2) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_type_ === nothing && Thrift.missingfield(:PageHeader, :type) + f_uncompressed_page_size === nothing && Thrift.missingfield(:PageHeader, :uncompressed_page_size) + f_compressed_page_size === nothing && Thrift.missingfield(:PageHeader, :compressed_page_size) + return PageHeader(f_type_, f_uncompressed_page_size, f_compressed_page_size, f_crc, f_data_page_header, f_index_page_header, f_dictionary_page_header, f_data_page_header_v2, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::PageHeader) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.type_.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, x.uncompressed_page_size) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I32) + Thrift.writei32!(w, x.compressed_page_size) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_crc = x.crc + if value_crc !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.I32) + Thrift.writei32!(w, value_crc) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_data_page_header = x.data_page_header + if value_data_page_header !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.STRUCT) + Thrift.encode!(w, value_data_page_header) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_index_page_header = x.index_page_header + if value_index_page_header !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.STRUCT) + Thrift.encode!(w, value_index_page_header) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_dictionary_page_header = x.dictionary_page_header + if value_dictionary_page_header !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.STRUCT) + Thrift.encode!(w, value_dictionary_page_header) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_data_page_header_v2 = x.data_page_header_v2 + if value_data_page_header_v2 !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.STRUCT) + Thrift.encode!(w, value_data_page_header_v2) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct KeyValue +Base.@kwdef struct KeyValue + key::String # 1: required string key + value::Union{Nothing, String} = nothing # 2: optional string value + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::KeyValue, b::KeyValue) + return a.key == b.key && a.value == b.value && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::KeyValue, b::KeyValue) + return isequal(a.key, b.key) && isequal(a.value, b.value) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::KeyValue, h::UInt) + h = hash(:KeyValue, h) + h = hash(x.key, h) + h = hash(x.value, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{KeyValue}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{KeyValue}) + return Thrift.decode(r, KeyValue) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::KeyValue) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{KeyValue}) + Thrift.enter!(r) + f_key = nothing + f_value = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BINARY + f_key = Thrift.readstring(r) + elseif id == Int16(2) && ty == Thrift.BINARY + f_value = Thrift.readstring(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_key === nothing && Thrift.missingfield(:KeyValue, :key) + return KeyValue(f_key, f_value, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::KeyValue) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BINARY) + Thrift.writestring!(w, x.key) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_value = x.value + if value_value !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.BINARY) + Thrift.writestring!(w, value_value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct SortingColumn +Base.@kwdef struct SortingColumn + column_idx::Int32 # 1: required i32 column_idx + descending::Bool # 2: required bool descending + nulls_first::Bool # 3: required bool nulls_first + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::SortingColumn, b::SortingColumn) + return a.column_idx == b.column_idx && a.descending == b.descending && a.nulls_first == b.nulls_first && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::SortingColumn, b::SortingColumn) + return isequal(a.column_idx, b.column_idx) && isequal(a.descending, b.descending) && isequal(a.nulls_first, b.nulls_first) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::SortingColumn, h::UInt) + h = hash(:SortingColumn, h) + h = hash(x.column_idx, h) + h = hash(x.descending, h) + h = hash(x.nulls_first, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{SortingColumn}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{SortingColumn}) + return Thrift.decode(r, SortingColumn) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::SortingColumn) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{SortingColumn}) + Thrift.enter!(r) + f_column_idx = nothing + f_descending = nothing + f_nulls_first = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_column_idx = Thrift.readi32(r) + elseif id == Int16(2) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_descending = ty == Thrift.BOOL_TRUE + elseif id == Int16(3) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_nulls_first = ty == Thrift.BOOL_TRUE + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_column_idx === nothing && Thrift.missingfield(:SortingColumn, :column_idx) + f_descending === nothing && Thrift.missingfield(:SortingColumn, :descending) + f_nulls_first === nothing && Thrift.missingfield(:SortingColumn, :nulls_first) + return SortingColumn(f_column_idx, f_descending, f_nulls_first, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::SortingColumn) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.column_idx) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), x.descending ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), x.nulls_first ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct PageEncodingStats +Base.@kwdef struct PageEncodingStats + page_type::PageType.T # 1: required PageType page_type + encoding::Encoding.T # 2: required Encoding encoding + count::Int32 # 3: required i32 count + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::PageEncodingStats, b::PageEncodingStats) + return a.page_type == b.page_type && a.encoding == b.encoding && a.count == b.count && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::PageEncodingStats, b::PageEncodingStats) + return isequal(a.page_type, b.page_type) && isequal(a.encoding, b.encoding) && isequal(a.count, b.count) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::PageEncodingStats, h::UInt) + h = hash(:PageEncodingStats, h) + h = hash(x.page_type, h) + h = hash(x.encoding, h) + h = hash(x.count, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{PageEncodingStats}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{PageEncodingStats}) + return Thrift.decode(r, PageEncodingStats) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::PageEncodingStats) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{PageEncodingStats}) + Thrift.enter!(r) + f_page_type = nothing + f_encoding = nothing + f_count = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_page_type = PageType.T(Thrift.readi32(r)) + elseif id == Int16(2) && ty == Thrift.I32 + f_encoding = Encoding.T(Thrift.readi32(r)) + elseif id == Int16(3) && ty == Thrift.I32 + f_count = Thrift.readi32(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_page_type === nothing && Thrift.missingfield(:PageEncodingStats, :page_type) + f_encoding === nothing && Thrift.missingfield(:PageEncodingStats, :encoding) + f_count === nothing && Thrift.missingfield(:PageEncodingStats, :count) + return PageEncodingStats(f_page_type, f_encoding, f_count, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::PageEncodingStats) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.page_type.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, x.encoding.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I32) + Thrift.writei32!(w, x.count) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct ColumnMetaData +Base.@kwdef struct ColumnMetaData + type_::Type.T # 1: required Type type + encodings::Vector{Encoding.T} # 2: required list encodings + path_in_schema::Vector{String} # 3: required list path_in_schema + codec::CompressionCodec.T # 4: required CompressionCodec codec + num_values::Int64 # 5: required i64 num_values + total_uncompressed_size::Int64 # 6: required i64 total_uncompressed_size + total_compressed_size::Int64 # 7: required i64 total_compressed_size + key_value_metadata::Union{Nothing, Vector{KeyValue}} = nothing # 8: optional list key_value_metadata + data_page_offset::Int64 # 9: required i64 data_page_offset + index_page_offset::Union{Nothing, Int64} = nothing # 10: optional i64 index_page_offset + dictionary_page_offset::Union{Nothing, Int64} = nothing # 11: optional i64 dictionary_page_offset + statistics::Union{Nothing, Statistics} = nothing # 12: optional Statistics statistics + encoding_stats::Union{Nothing, Vector{PageEncodingStats}} = nothing # 13: optional list encoding_stats + bloom_filter_offset::Union{Nothing, Int64} = nothing # 14: optional i64 bloom_filter_offset + bloom_filter_length::Union{Nothing, Int32} = nothing # 15: optional i32 bloom_filter_length + size_statistics::Union{Nothing, SizeStatistics} = nothing # 16: optional SizeStatistics size_statistics + geospatial_statistics::Union{Nothing, GeospatialStatistics} = nothing # 17: optional GeospatialStatistics geospatial_statistics + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::ColumnMetaData, b::ColumnMetaData) + return a.type_ == b.type_ && a.encodings == b.encodings && a.path_in_schema == b.path_in_schema && a.codec == b.codec && a.num_values == b.num_values && a.total_uncompressed_size == b.total_uncompressed_size && a.total_compressed_size == b.total_compressed_size && a.key_value_metadata == b.key_value_metadata && a.data_page_offset == b.data_page_offset && a.index_page_offset == b.index_page_offset && a.dictionary_page_offset == b.dictionary_page_offset && a.statistics == b.statistics && a.encoding_stats == b.encoding_stats && a.bloom_filter_offset == b.bloom_filter_offset && a.bloom_filter_length == b.bloom_filter_length && a.size_statistics == b.size_statistics && a.geospatial_statistics == b.geospatial_statistics && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::ColumnMetaData, b::ColumnMetaData) + return isequal(a.type_, b.type_) && isequal(a.encodings, b.encodings) && isequal(a.path_in_schema, b.path_in_schema) && isequal(a.codec, b.codec) && isequal(a.num_values, b.num_values) && isequal(a.total_uncompressed_size, b.total_uncompressed_size) && isequal(a.total_compressed_size, b.total_compressed_size) && isequal(a.key_value_metadata, b.key_value_metadata) && isequal(a.data_page_offset, b.data_page_offset) && isequal(a.index_page_offset, b.index_page_offset) && isequal(a.dictionary_page_offset, b.dictionary_page_offset) && isequal(a.statistics, b.statistics) && isequal(a.encoding_stats, b.encoding_stats) && isequal(a.bloom_filter_offset, b.bloom_filter_offset) && isequal(a.bloom_filter_length, b.bloom_filter_length) && isequal(a.size_statistics, b.size_statistics) && isequal(a.geospatial_statistics, b.geospatial_statistics) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::ColumnMetaData, h::UInt) + h = hash(:ColumnMetaData, h) + h = hash(x.type_, h) + h = hash(x.encodings, h) + h = hash(x.path_in_schema, h) + h = hash(x.codec, h) + h = hash(x.num_values, h) + h = hash(x.total_uncompressed_size, h) + h = hash(x.total_compressed_size, h) + h = hash(x.key_value_metadata, h) + h = hash(x.data_page_offset, h) + h = hash(x.index_page_offset, h) + h = hash(x.dictionary_page_offset, h) + h = hash(x.statistics, h) + h = hash(x.encoding_stats, h) + h = hash(x.bloom_filter_offset, h) + h = hash(x.bloom_filter_length, h) + h = hash(x.size_statistics, h) + h = hash(x.geospatial_statistics, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{ColumnMetaData}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{ColumnMetaData}) + return Thrift.decode(r, ColumnMetaData) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::ColumnMetaData) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{ColumnMetaData}) + Thrift.enter!(r) + f_type_ = nothing + f_encodings = nothing + f_path_in_schema = nothing + f_codec = nothing + f_num_values = nothing + f_total_uncompressed_size = nothing + f_total_compressed_size = nothing + f_key_value_metadata = nothing + f_data_page_offset = nothing + f_index_page_offset = nothing + f_dictionary_page_offset = nothing + f_statistics = nothing + f_encoding_stats = nothing + f_bloom_filter_offset = nothing + f_bloom_filter_length = nothing + f_size_statistics = nothing + f_geospatial_statistics = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_type_ = Type.T(Thrift.readi32(r)) + elseif id == Int16(2) && ty == Thrift.LIST + value_encodings = Thrift.readlist(r, Encoding.T) + if value_encodings === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_encodings = value_encodings + end + elseif id == Int16(3) && ty == Thrift.LIST + value_path_in_schema = Thrift.readlist(r, String) + if value_path_in_schema === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_path_in_schema = value_path_in_schema + end + elseif id == Int16(4) && ty == Thrift.I32 + f_codec = CompressionCodec.T(Thrift.readi32(r)) + elseif id == Int16(5) && ty == Thrift.I64 + f_num_values = Thrift.readi64(r) + elseif id == Int16(6) && ty == Thrift.I64 + f_total_uncompressed_size = Thrift.readi64(r) + elseif id == Int16(7) && ty == Thrift.I64 + f_total_compressed_size = Thrift.readi64(r) + elseif id == Int16(8) && ty == Thrift.LIST + value_key_value_metadata = Thrift.readlist(r, KeyValue) + if value_key_value_metadata === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_key_value_metadata = value_key_value_metadata + end + elseif id == Int16(9) && ty == Thrift.I64 + f_data_page_offset = Thrift.readi64(r) + elseif id == Int16(10) && ty == Thrift.I64 + f_index_page_offset = Thrift.readi64(r) + elseif id == Int16(11) && ty == Thrift.I64 + f_dictionary_page_offset = Thrift.readi64(r) + elseif id == Int16(12) && ty == Thrift.STRUCT + f_statistics = Thrift.decode(r, Statistics) + elseif id == Int16(13) && ty == Thrift.LIST + value_encoding_stats = Thrift.readlist(r, PageEncodingStats) + if value_encoding_stats === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_encoding_stats = value_encoding_stats + end + elseif id == Int16(14) && ty == Thrift.I64 + f_bloom_filter_offset = Thrift.readi64(r) + elseif id == Int16(15) && ty == Thrift.I32 + f_bloom_filter_length = Thrift.readi32(r) + elseif id == Int16(16) && ty == Thrift.STRUCT + f_size_statistics = Thrift.decode(r, SizeStatistics) + elseif id == Int16(17) && ty == Thrift.STRUCT + f_geospatial_statistics = Thrift.decode(r, GeospatialStatistics) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_type_ === nothing && Thrift.missingfield(:ColumnMetaData, :type) + f_encodings === nothing && Thrift.missingfield(:ColumnMetaData, :encodings) + f_path_in_schema === nothing && Thrift.missingfield(:ColumnMetaData, :path_in_schema) + f_codec === nothing && Thrift.missingfield(:ColumnMetaData, :codec) + f_num_values === nothing && Thrift.missingfield(:ColumnMetaData, :num_values) + f_total_uncompressed_size === nothing && Thrift.missingfield(:ColumnMetaData, :total_uncompressed_size) + f_total_compressed_size === nothing && Thrift.missingfield(:ColumnMetaData, :total_compressed_size) + f_data_page_offset === nothing && Thrift.missingfield(:ColumnMetaData, :data_page_offset) + return ColumnMetaData(f_type_, f_encodings, f_path_in_schema, f_codec, f_num_values, f_total_uncompressed_size, f_total_compressed_size, f_key_value_metadata, f_data_page_offset, f_index_page_offset, f_dictionary_page_offset, f_statistics, f_encoding_stats, f_bloom_filter_offset, f_bloom_filter_length, f_size_statistics, f_geospatial_statistics, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::ColumnMetaData) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.type_.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.LIST) + Thrift.writelist!(w, x.encodings) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.LIST) + Thrift.writelist!(w, x.path_in_schema) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.I32) + Thrift.writei32!(w, x.codec.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.I64) + Thrift.writei64!(w, x.num_values) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.I64) + Thrift.writei64!(w, x.total_uncompressed_size) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.I64) + Thrift.writei64!(w, x.total_compressed_size) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_key_value_metadata = x.key_value_metadata + if value_key_value_metadata !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.LIST) + Thrift.writelist!(w, value_key_value_metadata) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + lastid = Thrift.writefieldheader!(w, lastid, Int16(9), Thrift.I64) + Thrift.writei64!(w, x.data_page_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_index_page_offset = x.index_page_offset + if value_index_page_offset !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(10), Thrift.I64) + Thrift.writei64!(w, value_index_page_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_dictionary_page_offset = x.dictionary_page_offset + if value_dictionary_page_offset !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(11), Thrift.I64) + Thrift.writei64!(w, value_dictionary_page_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_statistics = x.statistics + if value_statistics !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(12), Thrift.STRUCT) + Thrift.encode!(w, value_statistics) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_encoding_stats = x.encoding_stats + if value_encoding_stats !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(13), Thrift.LIST) + Thrift.writelist!(w, value_encoding_stats) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_bloom_filter_offset = x.bloom_filter_offset + if value_bloom_filter_offset !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(14), Thrift.I64) + Thrift.writei64!(w, value_bloom_filter_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_bloom_filter_length = x.bloom_filter_length + if value_bloom_filter_length !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(15), Thrift.I32) + Thrift.writei32!(w, value_bloom_filter_length) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_size_statistics = x.size_statistics + if value_size_statistics !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(16), Thrift.STRUCT) + Thrift.encode!(w, value_size_statistics) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_geospatial_statistics = x.geospatial_statistics + if value_geospatial_statistics !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(17), Thrift.STRUCT) + Thrift.encode!(w, value_geospatial_statistics) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct EncryptionWithFooterKey +Base.@kwdef struct EncryptionWithFooterKey + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::EncryptionWithFooterKey, b::EncryptionWithFooterKey) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::EncryptionWithFooterKey, b::EncryptionWithFooterKey) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::EncryptionWithFooterKey, h::UInt) + h = hash(:EncryptionWithFooterKey, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{EncryptionWithFooterKey}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{EncryptionWithFooterKey}) + return Thrift.decode(r, EncryptionWithFooterKey) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::EncryptionWithFooterKey) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{EncryptionWithFooterKey}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return EncryptionWithFooterKey(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::EncryptionWithFooterKey) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct EncryptionWithColumnKey +Base.@kwdef struct EncryptionWithColumnKey + path_in_schema::Vector{String} # 1: required list path_in_schema + key_metadata::Union{Nothing, Vector{UInt8}} = nothing # 2: optional binary key_metadata + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::EncryptionWithColumnKey, b::EncryptionWithColumnKey) + return a.path_in_schema == b.path_in_schema && a.key_metadata == b.key_metadata && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::EncryptionWithColumnKey, b::EncryptionWithColumnKey) + return isequal(a.path_in_schema, b.path_in_schema) && isequal(a.key_metadata, b.key_metadata) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::EncryptionWithColumnKey, h::UInt) + h = hash(:EncryptionWithColumnKey, h) + h = hash(x.path_in_schema, h) + h = hash(x.key_metadata, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{EncryptionWithColumnKey}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{EncryptionWithColumnKey}) + return Thrift.decode(r, EncryptionWithColumnKey) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::EncryptionWithColumnKey) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{EncryptionWithColumnKey}) + Thrift.enter!(r) + f_path_in_schema = nothing + f_key_metadata = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.LIST + value_path_in_schema = Thrift.readlist(r, String) + if value_path_in_schema === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_path_in_schema = value_path_in_schema + end + elseif id == Int16(2) && ty == Thrift.BINARY + f_key_metadata = Thrift.readbinary(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_path_in_schema === nothing && Thrift.missingfield(:EncryptionWithColumnKey, :path_in_schema) + return EncryptionWithColumnKey(f_path_in_schema, f_key_metadata, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::EncryptionWithColumnKey) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.LIST) + Thrift.writelist!(w, x.path_in_schema) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_key_metadata = x.key_metadata + if value_key_metadata !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.BINARY) + Thrift.writebinary!(w, value_key_metadata) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union ColumnCryptoMetaData +struct ColumnCryptoMetaData + ENCRYPTION_WITH_FOOTER_KEY::Union{Nothing, EncryptionWithFooterKey} # 1: EncryptionWithFooterKey ENCRYPTION_WITH_FOOTER_KEY + ENCRYPTION_WITH_COLUMN_KEY::Union{Nothing, EncryptionWithColumnKey} # 2: EncryptionWithColumnKey ENCRYPTION_WITH_COLUMN_KEY + unknown_fields::Vector{Thrift.RawField} + function ColumnCryptoMetaData(ENCRYPTION_WITH_FOOTER_KEY, ENCRYPTION_WITH_COLUMN_KEY, unknown_fields) + Thrift.checkunionargs(:ColumnCryptoMetaData, (ENCRYPTION_WITH_FOOTER_KEY !== nothing) + (ENCRYPTION_WITH_COLUMN_KEY !== nothing), unknown_fields) + return new(ENCRYPTION_WITH_FOOTER_KEY, ENCRYPTION_WITH_COLUMN_KEY, unknown_fields) + end +end + +function ColumnCryptoMetaData(; ENCRYPTION_WITH_FOOTER_KEY=nothing, ENCRYPTION_WITH_COLUMN_KEY=nothing, unknown_fields=Thrift.RawField[]) + return ColumnCryptoMetaData(ENCRYPTION_WITH_FOOTER_KEY, ENCRYPTION_WITH_COLUMN_KEY, unknown_fields) +end + +function Base.:(==)(a::ColumnCryptoMetaData, b::ColumnCryptoMetaData) + return a.ENCRYPTION_WITH_FOOTER_KEY == b.ENCRYPTION_WITH_FOOTER_KEY && a.ENCRYPTION_WITH_COLUMN_KEY == b.ENCRYPTION_WITH_COLUMN_KEY && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::ColumnCryptoMetaData, b::ColumnCryptoMetaData) + return isequal(a.ENCRYPTION_WITH_FOOTER_KEY, b.ENCRYPTION_WITH_FOOTER_KEY) && isequal(a.ENCRYPTION_WITH_COLUMN_KEY, b.ENCRYPTION_WITH_COLUMN_KEY) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::ColumnCryptoMetaData, h::UInt) + h = hash(:ColumnCryptoMetaData, h) + h = hash(x.ENCRYPTION_WITH_FOOTER_KEY, h) + h = hash(x.ENCRYPTION_WITH_COLUMN_KEY, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{ColumnCryptoMetaData}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{ColumnCryptoMetaData}) + return Thrift.decode(r, ColumnCryptoMetaData) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::ColumnCryptoMetaData) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{ColumnCryptoMetaData}) + Thrift.enter!(r) + f_ENCRYPTION_WITH_FOOTER_KEY = nothing + f_ENCRYPTION_WITH_COLUMN_KEY = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_ENCRYPTION_WITH_FOOTER_KEY = Thrift.decode(r, EncryptionWithFooterKey) + elseif id == Int16(2) && ty == Thrift.STRUCT + f_ENCRYPTION_WITH_COLUMN_KEY = Thrift.decode(r, EncryptionWithColumnKey) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:ColumnCryptoMetaData, (f_ENCRYPTION_WITH_FOOTER_KEY !== nothing) + (f_ENCRYPTION_WITH_COLUMN_KEY !== nothing), unknown_fields) + return ColumnCryptoMetaData(f_ENCRYPTION_WITH_FOOTER_KEY, f_ENCRYPTION_WITH_COLUMN_KEY, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::ColumnCryptoMetaData) + unknown = x.unknown_fields + Thrift.checkunionargs(:ColumnCryptoMetaData, (x.ENCRYPTION_WITH_FOOTER_KEY !== nothing) + (x.ENCRYPTION_WITH_COLUMN_KEY !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_ENCRYPTION_WITH_FOOTER_KEY = x.ENCRYPTION_WITH_FOOTER_KEY + if value_ENCRYPTION_WITH_FOOTER_KEY !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_ENCRYPTION_WITH_FOOTER_KEY) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_ENCRYPTION_WITH_COLUMN_KEY = x.ENCRYPTION_WITH_COLUMN_KEY + if value_ENCRYPTION_WITH_COLUMN_KEY !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, value_ENCRYPTION_WITH_COLUMN_KEY) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct ColumnChunk +Base.@kwdef struct ColumnChunk + file_path::Union{Nothing, String} = nothing # 1: optional string file_path + file_offset::Int64 = Int64(0) # 2: required i64 file_offset = 0 + meta_data::Union{Nothing, ColumnMetaData} = nothing # 3: optional ColumnMetaData meta_data + offset_index_offset::Union{Nothing, Int64} = nothing # 4: optional i64 offset_index_offset + offset_index_length::Union{Nothing, Int32} = nothing # 5: optional i32 offset_index_length + column_index_offset::Union{Nothing, Int64} = nothing # 6: optional i64 column_index_offset + column_index_length::Union{Nothing, Int32} = nothing # 7: optional i32 column_index_length + crypto_metadata::Union{Nothing, ColumnCryptoMetaData} = nothing # 8: optional ColumnCryptoMetaData crypto_metadata + encrypted_column_metadata::Union{Nothing, Vector{UInt8}} = nothing # 9: optional binary encrypted_column_metadata + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::ColumnChunk, b::ColumnChunk) + return a.file_path == b.file_path && a.file_offset == b.file_offset && a.meta_data == b.meta_data && a.offset_index_offset == b.offset_index_offset && a.offset_index_length == b.offset_index_length && a.column_index_offset == b.column_index_offset && a.column_index_length == b.column_index_length && a.crypto_metadata == b.crypto_metadata && a.encrypted_column_metadata == b.encrypted_column_metadata && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::ColumnChunk, b::ColumnChunk) + return isequal(a.file_path, b.file_path) && isequal(a.file_offset, b.file_offset) && isequal(a.meta_data, b.meta_data) && isequal(a.offset_index_offset, b.offset_index_offset) && isequal(a.offset_index_length, b.offset_index_length) && isequal(a.column_index_offset, b.column_index_offset) && isequal(a.column_index_length, b.column_index_length) && isequal(a.crypto_metadata, b.crypto_metadata) && isequal(a.encrypted_column_metadata, b.encrypted_column_metadata) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::ColumnChunk, h::UInt) + h = hash(:ColumnChunk, h) + h = hash(x.file_path, h) + h = hash(x.file_offset, h) + h = hash(x.meta_data, h) + h = hash(x.offset_index_offset, h) + h = hash(x.offset_index_length, h) + h = hash(x.column_index_offset, h) + h = hash(x.column_index_length, h) + h = hash(x.crypto_metadata, h) + h = hash(x.encrypted_column_metadata, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{ColumnChunk}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{ColumnChunk}) + return Thrift.decode(r, ColumnChunk) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::ColumnChunk) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{ColumnChunk}) + Thrift.enter!(r) + f_file_path = nothing + f_file_offset = Int64(0) + f_meta_data = nothing + f_offset_index_offset = nothing + f_offset_index_length = nothing + f_column_index_offset = nothing + f_column_index_length = nothing + f_crypto_metadata = nothing + f_encrypted_column_metadata = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BINARY + f_file_path = Thrift.readstring(r) + elseif id == Int16(2) && ty == Thrift.I64 + f_file_offset = Thrift.readi64(r) + elseif id == Int16(3) && ty == Thrift.STRUCT + f_meta_data = Thrift.decode(r, ColumnMetaData) + elseif id == Int16(4) && ty == Thrift.I64 + f_offset_index_offset = Thrift.readi64(r) + elseif id == Int16(5) && ty == Thrift.I32 + f_offset_index_length = Thrift.readi32(r) + elseif id == Int16(6) && ty == Thrift.I64 + f_column_index_offset = Thrift.readi64(r) + elseif id == Int16(7) && ty == Thrift.I32 + f_column_index_length = Thrift.readi32(r) + elseif id == Int16(8) && ty == Thrift.STRUCT + f_crypto_metadata = Thrift.decode(r, ColumnCryptoMetaData) + elseif id == Int16(9) && ty == Thrift.BINARY + f_encrypted_column_metadata = Thrift.readbinary(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return ColumnChunk(f_file_path, f_file_offset, f_meta_data, f_offset_index_offset, f_offset_index_length, f_column_index_offset, f_column_index_length, f_crypto_metadata, f_encrypted_column_metadata, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::ColumnChunk) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_file_path = x.file_path + if value_file_path !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BINARY) + Thrift.writestring!(w, value_file_path) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I64) + Thrift.writei64!(w, x.file_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_meta_data = x.meta_data + if value_meta_data !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.STRUCT) + Thrift.encode!(w, value_meta_data) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_offset_index_offset = x.offset_index_offset + if value_offset_index_offset !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.I64) + Thrift.writei64!(w, value_offset_index_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_offset_index_length = x.offset_index_length + if value_offset_index_length !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.I32) + Thrift.writei32!(w, value_offset_index_length) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_column_index_offset = x.column_index_offset + if value_column_index_offset !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.I64) + Thrift.writei64!(w, value_column_index_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_column_index_length = x.column_index_length + if value_column_index_length !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.I32) + Thrift.writei32!(w, value_column_index_length) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_crypto_metadata = x.crypto_metadata + if value_crypto_metadata !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.STRUCT) + Thrift.encode!(w, value_crypto_metadata) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_encrypted_column_metadata = x.encrypted_column_metadata + if value_encrypted_column_metadata !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(9), Thrift.BINARY) + Thrift.writebinary!(w, value_encrypted_column_metadata) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct RowGroup +Base.@kwdef struct RowGroup + columns::Vector{ColumnChunk} # 1: required list columns + total_byte_size::Int64 # 2: required i64 total_byte_size + num_rows::Int64 # 3: required i64 num_rows + sorting_columns::Union{Nothing, Vector{SortingColumn}} = nothing # 4: optional list sorting_columns + file_offset::Union{Nothing, Int64} = nothing # 5: optional i64 file_offset + total_compressed_size::Union{Nothing, Int64} = nothing # 6: optional i64 total_compressed_size + ordinal::Union{Nothing, Int16} = nothing # 7: optional i16 ordinal + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::RowGroup, b::RowGroup) + return a.columns == b.columns && a.total_byte_size == b.total_byte_size && a.num_rows == b.num_rows && a.sorting_columns == b.sorting_columns && a.file_offset == b.file_offset && a.total_compressed_size == b.total_compressed_size && a.ordinal == b.ordinal && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::RowGroup, b::RowGroup) + return isequal(a.columns, b.columns) && isequal(a.total_byte_size, b.total_byte_size) && isequal(a.num_rows, b.num_rows) && isequal(a.sorting_columns, b.sorting_columns) && isequal(a.file_offset, b.file_offset) && isequal(a.total_compressed_size, b.total_compressed_size) && isequal(a.ordinal, b.ordinal) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::RowGroup, h::UInt) + h = hash(:RowGroup, h) + h = hash(x.columns, h) + h = hash(x.total_byte_size, h) + h = hash(x.num_rows, h) + h = hash(x.sorting_columns, h) + h = hash(x.file_offset, h) + h = hash(x.total_compressed_size, h) + h = hash(x.ordinal, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{RowGroup}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{RowGroup}) + return Thrift.decode(r, RowGroup) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::RowGroup) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{RowGroup}) + Thrift.enter!(r) + f_columns = nothing + f_total_byte_size = nothing + f_num_rows = nothing + f_sorting_columns = nothing + f_file_offset = nothing + f_total_compressed_size = nothing + f_ordinal = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.LIST + value_columns = Thrift.readlist(r, ColumnChunk) + if value_columns === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_columns = value_columns + end + elseif id == Int16(2) && ty == Thrift.I64 + f_total_byte_size = Thrift.readi64(r) + elseif id == Int16(3) && ty == Thrift.I64 + f_num_rows = Thrift.readi64(r) + elseif id == Int16(4) && ty == Thrift.LIST + value_sorting_columns = Thrift.readlist(r, SortingColumn) + if value_sorting_columns === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_sorting_columns = value_sorting_columns + end + elseif id == Int16(5) && ty == Thrift.I64 + f_file_offset = Thrift.readi64(r) + elseif id == Int16(6) && ty == Thrift.I64 + f_total_compressed_size = Thrift.readi64(r) + elseif id == Int16(7) && ty == Thrift.I16 + f_ordinal = Thrift.readi16(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_columns === nothing && Thrift.missingfield(:RowGroup, :columns) + f_total_byte_size === nothing && Thrift.missingfield(:RowGroup, :total_byte_size) + f_num_rows === nothing && Thrift.missingfield(:RowGroup, :num_rows) + return RowGroup(f_columns, f_total_byte_size, f_num_rows, f_sorting_columns, f_file_offset, f_total_compressed_size, f_ordinal, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::RowGroup) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.LIST) + Thrift.writelist!(w, x.columns) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I64) + Thrift.writei64!(w, x.total_byte_size) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I64) + Thrift.writei64!(w, x.num_rows) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_sorting_columns = x.sorting_columns + if value_sorting_columns !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.LIST) + Thrift.writelist!(w, value_sorting_columns) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_file_offset = x.file_offset + if value_file_offset !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.I64) + Thrift.writei64!(w, value_file_offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_total_compressed_size = x.total_compressed_size + if value_total_compressed_size !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.I64) + Thrift.writei64!(w, value_total_compressed_size) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_ordinal = x.ordinal + if value_ordinal !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.I16) + Thrift.writei16!(w, value_ordinal) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct TypeDefinedOrder +Base.@kwdef struct TypeDefinedOrder + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::TypeDefinedOrder, b::TypeDefinedOrder) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::TypeDefinedOrder, b::TypeDefinedOrder) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::TypeDefinedOrder, h::UInt) + h = hash(:TypeDefinedOrder, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{TypeDefinedOrder}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{TypeDefinedOrder}) + return Thrift.decode(r, TypeDefinedOrder) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::TypeDefinedOrder) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{TypeDefinedOrder}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return TypeDefinedOrder(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::TypeDefinedOrder) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct IEEE754TotalOrder +Base.@kwdef struct IEEE754TotalOrder + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::IEEE754TotalOrder, b::IEEE754TotalOrder) + return a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::IEEE754TotalOrder, b::IEEE754TotalOrder) + return isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::IEEE754TotalOrder, h::UInt) + h = hash(:IEEE754TotalOrder, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{IEEE754TotalOrder}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{IEEE754TotalOrder}) + return Thrift.decode(r, IEEE754TotalOrder) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::IEEE754TotalOrder) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{IEEE754TotalOrder}) + Thrift.enter!(r) + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return IEEE754TotalOrder(unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::IEEE754TotalOrder) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union ColumnOrder +struct ColumnOrder + TYPE_ORDER::Union{Nothing, TypeDefinedOrder} # 1: TypeDefinedOrder TYPE_ORDER + IEEE_754_TOTAL_ORDER::Union{Nothing, IEEE754TotalOrder} # 2: IEEE754TotalOrder IEEE_754_TOTAL_ORDER + unknown_fields::Vector{Thrift.RawField} + function ColumnOrder(TYPE_ORDER, IEEE_754_TOTAL_ORDER, unknown_fields) + Thrift.checkunionargs(:ColumnOrder, (TYPE_ORDER !== nothing) + (IEEE_754_TOTAL_ORDER !== nothing), unknown_fields) + return new(TYPE_ORDER, IEEE_754_TOTAL_ORDER, unknown_fields) + end +end + +function ColumnOrder(; TYPE_ORDER=nothing, IEEE_754_TOTAL_ORDER=nothing, unknown_fields=Thrift.RawField[]) + return ColumnOrder(TYPE_ORDER, IEEE_754_TOTAL_ORDER, unknown_fields) +end + +function Base.:(==)(a::ColumnOrder, b::ColumnOrder) + return a.TYPE_ORDER == b.TYPE_ORDER && a.IEEE_754_TOTAL_ORDER == b.IEEE_754_TOTAL_ORDER && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::ColumnOrder, b::ColumnOrder) + return isequal(a.TYPE_ORDER, b.TYPE_ORDER) && isequal(a.IEEE_754_TOTAL_ORDER, b.IEEE_754_TOTAL_ORDER) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::ColumnOrder, h::UInt) + h = hash(:ColumnOrder, h) + h = hash(x.TYPE_ORDER, h) + h = hash(x.IEEE_754_TOTAL_ORDER, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{ColumnOrder}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{ColumnOrder}) + return Thrift.decode(r, ColumnOrder) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::ColumnOrder) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{ColumnOrder}) + Thrift.enter!(r) + f_TYPE_ORDER = nothing + f_IEEE_754_TOTAL_ORDER = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_TYPE_ORDER = Thrift.decode(r, TypeDefinedOrder) + elseif id == Int16(2) && ty == Thrift.STRUCT + f_IEEE_754_TOTAL_ORDER = Thrift.decode(r, IEEE754TotalOrder) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:ColumnOrder, (f_TYPE_ORDER !== nothing) + (f_IEEE_754_TOTAL_ORDER !== nothing), unknown_fields) + return ColumnOrder(f_TYPE_ORDER, f_IEEE_754_TOTAL_ORDER, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::ColumnOrder) + unknown = x.unknown_fields + Thrift.checkunionargs(:ColumnOrder, (x.TYPE_ORDER !== nothing) + (x.IEEE_754_TOTAL_ORDER !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_TYPE_ORDER = x.TYPE_ORDER + if value_TYPE_ORDER !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_TYPE_ORDER) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_IEEE_754_TOTAL_ORDER = x.IEEE_754_TOTAL_ORDER + if value_IEEE_754_TOTAL_ORDER !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, value_IEEE_754_TOTAL_ORDER) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct PageLocation +Base.@kwdef struct PageLocation + offset::Int64 # 1: required i64 offset + compressed_page_size::Int32 # 2: required i32 compressed_page_size + first_row_index::Int64 # 3: required i64 first_row_index + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::PageLocation, b::PageLocation) + return a.offset == b.offset && a.compressed_page_size == b.compressed_page_size && a.first_row_index == b.first_row_index && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::PageLocation, b::PageLocation) + return isequal(a.offset, b.offset) && isequal(a.compressed_page_size, b.compressed_page_size) && isequal(a.first_row_index, b.first_row_index) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::PageLocation, h::UInt) + h = hash(:PageLocation, h) + h = hash(x.offset, h) + h = hash(x.compressed_page_size, h) + h = hash(x.first_row_index, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{PageLocation}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{PageLocation}) + return Thrift.decode(r, PageLocation) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::PageLocation) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{PageLocation}) + Thrift.enter!(r) + f_offset = nothing + f_compressed_page_size = nothing + f_first_row_index = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I64 + f_offset = Thrift.readi64(r) + elseif id == Int16(2) && ty == Thrift.I32 + f_compressed_page_size = Thrift.readi32(r) + elseif id == Int16(3) && ty == Thrift.I64 + f_first_row_index = Thrift.readi64(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_offset === nothing && Thrift.missingfield(:PageLocation, :offset) + f_compressed_page_size === nothing && Thrift.missingfield(:PageLocation, :compressed_page_size) + f_first_row_index === nothing && Thrift.missingfield(:PageLocation, :first_row_index) + return PageLocation(f_offset, f_compressed_page_size, f_first_row_index, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::PageLocation) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I64) + Thrift.writei64!(w, x.offset) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.I32) + Thrift.writei32!(w, x.compressed_page_size) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I64) + Thrift.writei64!(w, x.first_row_index) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct OffsetIndex +Base.@kwdef struct OffsetIndex + page_locations::Vector{PageLocation} # 1: required list page_locations + unencoded_byte_array_data_bytes::Union{Nothing, Vector{Int64}} = nothing # 2: optional list unencoded_byte_array_data_bytes + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::OffsetIndex, b::OffsetIndex) + return a.page_locations == b.page_locations && a.unencoded_byte_array_data_bytes == b.unencoded_byte_array_data_bytes && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::OffsetIndex, b::OffsetIndex) + return isequal(a.page_locations, b.page_locations) && isequal(a.unencoded_byte_array_data_bytes, b.unencoded_byte_array_data_bytes) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::OffsetIndex, h::UInt) + h = hash(:OffsetIndex, h) + h = hash(x.page_locations, h) + h = hash(x.unencoded_byte_array_data_bytes, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{OffsetIndex}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{OffsetIndex}) + return Thrift.decode(r, OffsetIndex) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::OffsetIndex) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{OffsetIndex}) + Thrift.enter!(r) + f_page_locations = nothing + f_unencoded_byte_array_data_bytes = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.LIST + value_page_locations = Thrift.readlist(r, PageLocation) + if value_page_locations === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_page_locations = value_page_locations + end + elseif id == Int16(2) && ty == Thrift.LIST + value_unencoded_byte_array_data_bytes = Thrift.readlist(r, Int64) + if value_unencoded_byte_array_data_bytes === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_unencoded_byte_array_data_bytes = value_unencoded_byte_array_data_bytes + end + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_page_locations === nothing && Thrift.missingfield(:OffsetIndex, :page_locations) + return OffsetIndex(f_page_locations, f_unencoded_byte_array_data_bytes, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::OffsetIndex) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.LIST) + Thrift.writelist!(w, x.page_locations) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_unencoded_byte_array_data_bytes = x.unencoded_byte_array_data_bytes + if value_unencoded_byte_array_data_bytes !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.LIST) + Thrift.writelist!(w, value_unencoded_byte_array_data_bytes) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct ColumnIndex +Base.@kwdef struct ColumnIndex + null_pages::Vector{Bool} # 1: required list null_pages + min_values::Vector{Vector{UInt8}} # 2: required list min_values + max_values::Vector{Vector{UInt8}} # 3: required list max_values + boundary_order::BoundaryOrder.T # 4: required BoundaryOrder boundary_order + null_counts::Union{Nothing, Vector{Int64}} = nothing # 5: optional list null_counts + repetition_level_histograms::Union{Nothing, Vector{Int64}} = nothing # 6: optional list repetition_level_histograms + definition_level_histograms::Union{Nothing, Vector{Int64}} = nothing # 7: optional list definition_level_histograms + nan_counts::Union{Nothing, Vector{Int64}} = nothing # 8: optional list nan_counts + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::ColumnIndex, b::ColumnIndex) + return a.null_pages == b.null_pages && a.min_values == b.min_values && a.max_values == b.max_values && a.boundary_order == b.boundary_order && a.null_counts == b.null_counts && a.repetition_level_histograms == b.repetition_level_histograms && a.definition_level_histograms == b.definition_level_histograms && a.nan_counts == b.nan_counts && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::ColumnIndex, b::ColumnIndex) + return isequal(a.null_pages, b.null_pages) && isequal(a.min_values, b.min_values) && isequal(a.max_values, b.max_values) && isequal(a.boundary_order, b.boundary_order) && isequal(a.null_counts, b.null_counts) && isequal(a.repetition_level_histograms, b.repetition_level_histograms) && isequal(a.definition_level_histograms, b.definition_level_histograms) && isequal(a.nan_counts, b.nan_counts) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::ColumnIndex, h::UInt) + h = hash(:ColumnIndex, h) + h = hash(x.null_pages, h) + h = hash(x.min_values, h) + h = hash(x.max_values, h) + h = hash(x.boundary_order, h) + h = hash(x.null_counts, h) + h = hash(x.repetition_level_histograms, h) + h = hash(x.definition_level_histograms, h) + h = hash(x.nan_counts, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{ColumnIndex}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{ColumnIndex}) + return Thrift.decode(r, ColumnIndex) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::ColumnIndex) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{ColumnIndex}) + Thrift.enter!(r) + f_null_pages = nothing + f_min_values = nothing + f_max_values = nothing + f_boundary_order = nothing + f_null_counts = nothing + f_repetition_level_histograms = nothing + f_definition_level_histograms = nothing + f_nan_counts = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.LIST + value_null_pages = Thrift.readlist(r, Bool) + if value_null_pages === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_null_pages = value_null_pages + end + elseif id == Int16(2) && ty == Thrift.LIST + value_min_values = Thrift.readlist(r, Vector{UInt8}) + if value_min_values === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_min_values = value_min_values + end + elseif id == Int16(3) && ty == Thrift.LIST + value_max_values = Thrift.readlist(r, Vector{UInt8}) + if value_max_values === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_max_values = value_max_values + end + elseif id == Int16(4) && ty == Thrift.I32 + f_boundary_order = BoundaryOrder.T(Thrift.readi32(r)) + elseif id == Int16(5) && ty == Thrift.LIST + value_null_counts = Thrift.readlist(r, Int64) + if value_null_counts === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_null_counts = value_null_counts + end + elseif id == Int16(6) && ty == Thrift.LIST + value_repetition_level_histograms = Thrift.readlist(r, Int64) + if value_repetition_level_histograms === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_repetition_level_histograms = value_repetition_level_histograms + end + elseif id == Int16(7) && ty == Thrift.LIST + value_definition_level_histograms = Thrift.readlist(r, Int64) + if value_definition_level_histograms === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_definition_level_histograms = value_definition_level_histograms + end + elseif id == Int16(8) && ty == Thrift.LIST + value_nan_counts = Thrift.readlist(r, Int64) + if value_nan_counts === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_nan_counts = value_nan_counts + end + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_null_pages === nothing && Thrift.missingfield(:ColumnIndex, :null_pages) + f_min_values === nothing && Thrift.missingfield(:ColumnIndex, :min_values) + f_max_values === nothing && Thrift.missingfield(:ColumnIndex, :max_values) + f_boundary_order === nothing && Thrift.missingfield(:ColumnIndex, :boundary_order) + return ColumnIndex(f_null_pages, f_min_values, f_max_values, f_boundary_order, f_null_counts, f_repetition_level_histograms, f_definition_level_histograms, f_nan_counts, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::ColumnIndex) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.LIST) + Thrift.writelist!(w, x.null_pages) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.LIST) + Thrift.writelist!(w, x.min_values) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.LIST) + Thrift.writelist!(w, x.max_values) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.I32) + Thrift.writei32!(w, x.boundary_order.value) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_null_counts = x.null_counts + if value_null_counts !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.LIST) + Thrift.writelist!(w, value_null_counts) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_repetition_level_histograms = x.repetition_level_histograms + if value_repetition_level_histograms !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.LIST) + Thrift.writelist!(w, value_repetition_level_histograms) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_definition_level_histograms = x.definition_level_histograms + if value_definition_level_histograms !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.LIST) + Thrift.writelist!(w, value_definition_level_histograms) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_nan_counts = x.nan_counts + if value_nan_counts !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.LIST) + Thrift.writelist!(w, value_nan_counts) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct AesGcmV1 +Base.@kwdef struct AesGcmV1 + aad_prefix::Union{Nothing, Vector{UInt8}} = nothing # 1: optional binary aad_prefix + aad_file_unique::Union{Nothing, Vector{UInt8}} = nothing # 2: optional binary aad_file_unique + supply_aad_prefix::Union{Nothing, Bool} = nothing # 3: optional bool supply_aad_prefix + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::AesGcmV1, b::AesGcmV1) + return a.aad_prefix == b.aad_prefix && a.aad_file_unique == b.aad_file_unique && a.supply_aad_prefix == b.supply_aad_prefix && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::AesGcmV1, b::AesGcmV1) + return isequal(a.aad_prefix, b.aad_prefix) && isequal(a.aad_file_unique, b.aad_file_unique) && isequal(a.supply_aad_prefix, b.supply_aad_prefix) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::AesGcmV1, h::UInt) + h = hash(:AesGcmV1, h) + h = hash(x.aad_prefix, h) + h = hash(x.aad_file_unique, h) + h = hash(x.supply_aad_prefix, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{AesGcmV1}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{AesGcmV1}) + return Thrift.decode(r, AesGcmV1) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::AesGcmV1) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{AesGcmV1}) + Thrift.enter!(r) + f_aad_prefix = nothing + f_aad_file_unique = nothing + f_supply_aad_prefix = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BINARY + f_aad_prefix = Thrift.readbinary(r) + elseif id == Int16(2) && ty == Thrift.BINARY + f_aad_file_unique = Thrift.readbinary(r) + elseif id == Int16(3) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_supply_aad_prefix = ty == Thrift.BOOL_TRUE + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return AesGcmV1(f_aad_prefix, f_aad_file_unique, f_supply_aad_prefix, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::AesGcmV1) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_aad_prefix = x.aad_prefix + if value_aad_prefix !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BINARY) + Thrift.writebinary!(w, value_aad_prefix) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_aad_file_unique = x.aad_file_unique + if value_aad_file_unique !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.BINARY) + Thrift.writebinary!(w, value_aad_file_unique) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_supply_aad_prefix = x.supply_aad_prefix + if value_supply_aad_prefix !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), value_supply_aad_prefix ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct AesGcmCtrV1 +Base.@kwdef struct AesGcmCtrV1 + aad_prefix::Union{Nothing, Vector{UInt8}} = nothing # 1: optional binary aad_prefix + aad_file_unique::Union{Nothing, Vector{UInt8}} = nothing # 2: optional binary aad_file_unique + supply_aad_prefix::Union{Nothing, Bool} = nothing # 3: optional bool supply_aad_prefix + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::AesGcmCtrV1, b::AesGcmCtrV1) + return a.aad_prefix == b.aad_prefix && a.aad_file_unique == b.aad_file_unique && a.supply_aad_prefix == b.supply_aad_prefix && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::AesGcmCtrV1, b::AesGcmCtrV1) + return isequal(a.aad_prefix, b.aad_prefix) && isequal(a.aad_file_unique, b.aad_file_unique) && isequal(a.supply_aad_prefix, b.supply_aad_prefix) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::AesGcmCtrV1, h::UInt) + h = hash(:AesGcmCtrV1, h) + h = hash(x.aad_prefix, h) + h = hash(x.aad_file_unique, h) + h = hash(x.supply_aad_prefix, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{AesGcmCtrV1}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{AesGcmCtrV1}) + return Thrift.decode(r, AesGcmCtrV1) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::AesGcmCtrV1) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{AesGcmCtrV1}) + Thrift.enter!(r) + f_aad_prefix = nothing + f_aad_file_unique = nothing + f_supply_aad_prefix = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.BINARY + f_aad_prefix = Thrift.readbinary(r) + elseif id == Int16(2) && ty == Thrift.BINARY + f_aad_file_unique = Thrift.readbinary(r) + elseif id == Int16(3) && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE) + f_supply_aad_prefix = ty == Thrift.BOOL_TRUE + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + return AesGcmCtrV1(f_aad_prefix, f_aad_file_unique, f_supply_aad_prefix, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::AesGcmCtrV1) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_aad_prefix = x.aad_prefix + if value_aad_prefix !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.BINARY) + Thrift.writebinary!(w, value_aad_prefix) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_aad_file_unique = x.aad_file_unique + if value_aad_file_unique !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.BINARY) + Thrift.writebinary!(w, value_aad_file_unique) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_supply_aad_prefix = x.supply_aad_prefix + if value_supply_aad_prefix !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), value_supply_aad_prefix ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift union EncryptionAlgorithm +struct EncryptionAlgorithm + AES_GCM_V1::Union{Nothing, AesGcmV1} # 1: AesGcmV1 AES_GCM_V1 + AES_GCM_CTR_V1::Union{Nothing, AesGcmCtrV1} # 2: AesGcmCtrV1 AES_GCM_CTR_V1 + unknown_fields::Vector{Thrift.RawField} + function EncryptionAlgorithm(AES_GCM_V1, AES_GCM_CTR_V1, unknown_fields) + Thrift.checkunionargs(:EncryptionAlgorithm, (AES_GCM_V1 !== nothing) + (AES_GCM_CTR_V1 !== nothing), unknown_fields) + return new(AES_GCM_V1, AES_GCM_CTR_V1, unknown_fields) + end +end + +function EncryptionAlgorithm(; AES_GCM_V1=nothing, AES_GCM_CTR_V1=nothing, unknown_fields=Thrift.RawField[]) + return EncryptionAlgorithm(AES_GCM_V1, AES_GCM_CTR_V1, unknown_fields) +end + +function Base.:(==)(a::EncryptionAlgorithm, b::EncryptionAlgorithm) + return a.AES_GCM_V1 == b.AES_GCM_V1 && a.AES_GCM_CTR_V1 == b.AES_GCM_CTR_V1 && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::EncryptionAlgorithm, b::EncryptionAlgorithm) + return isequal(a.AES_GCM_V1, b.AES_GCM_V1) && isequal(a.AES_GCM_CTR_V1, b.AES_GCM_CTR_V1) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::EncryptionAlgorithm, h::UInt) + h = hash(:EncryptionAlgorithm, h) + h = hash(x.AES_GCM_V1, h) + h = hash(x.AES_GCM_CTR_V1, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{EncryptionAlgorithm}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{EncryptionAlgorithm}) + return Thrift.decode(r, EncryptionAlgorithm) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::EncryptionAlgorithm) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{EncryptionAlgorithm}) + Thrift.enter!(r) + f_AES_GCM_V1 = nothing + f_AES_GCM_CTR_V1 = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_AES_GCM_V1 = Thrift.decode(r, AesGcmV1) + elseif id == Int16(2) && ty == Thrift.STRUCT + f_AES_GCM_CTR_V1 = Thrift.decode(r, AesGcmCtrV1) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + Thrift.checkunion(:EncryptionAlgorithm, (f_AES_GCM_V1 !== nothing) + (f_AES_GCM_CTR_V1 !== nothing), unknown_fields) + return EncryptionAlgorithm(f_AES_GCM_V1, f_AES_GCM_CTR_V1, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::EncryptionAlgorithm) + unknown = x.unknown_fields + Thrift.checkunionargs(:EncryptionAlgorithm, (x.AES_GCM_V1 !== nothing) + (x.AES_GCM_CTR_V1 !== nothing), unknown) + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_AES_GCM_V1 = x.AES_GCM_V1 + if value_AES_GCM_V1 !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, value_AES_GCM_V1) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_AES_GCM_CTR_V1 = x.AES_GCM_CTR_V1 + if value_AES_GCM_CTR_V1 !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.STRUCT) + Thrift.encode!(w, value_AES_GCM_CTR_V1) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct FileMetaData +Base.@kwdef struct FileMetaData + version::Int32 # 1: required i32 version + schema::Vector{SchemaElement} # 2: required list schema + num_rows::Int64 # 3: required i64 num_rows + row_groups::Vector{RowGroup} # 4: required list row_groups + key_value_metadata::Union{Nothing, Vector{KeyValue}} = nothing # 5: optional list key_value_metadata + created_by::Union{Nothing, String} = nothing # 6: optional string created_by + column_orders::Union{Nothing, Vector{ColumnOrder}} = nothing # 7: optional list column_orders + encryption_algorithm::Union{Nothing, EncryptionAlgorithm} = nothing # 8: optional EncryptionAlgorithm encryption_algorithm + footer_signing_key_metadata::Union{Nothing, Vector{UInt8}} = nothing # 9: optional binary footer_signing_key_metadata + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::FileMetaData, b::FileMetaData) + return a.version == b.version && a.schema == b.schema && a.num_rows == b.num_rows && a.row_groups == b.row_groups && a.key_value_metadata == b.key_value_metadata && a.created_by == b.created_by && a.column_orders == b.column_orders && a.encryption_algorithm == b.encryption_algorithm && a.footer_signing_key_metadata == b.footer_signing_key_metadata && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::FileMetaData, b::FileMetaData) + return isequal(a.version, b.version) && isequal(a.schema, b.schema) && isequal(a.num_rows, b.num_rows) && isequal(a.row_groups, b.row_groups) && isequal(a.key_value_metadata, b.key_value_metadata) && isequal(a.created_by, b.created_by) && isequal(a.column_orders, b.column_orders) && isequal(a.encryption_algorithm, b.encryption_algorithm) && isequal(a.footer_signing_key_metadata, b.footer_signing_key_metadata) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::FileMetaData, h::UInt) + h = hash(:FileMetaData, h) + h = hash(x.version, h) + h = hash(x.schema, h) + h = hash(x.num_rows, h) + h = hash(x.row_groups, h) + h = hash(x.key_value_metadata, h) + h = hash(x.created_by, h) + h = hash(x.column_orders, h) + h = hash(x.encryption_algorithm, h) + h = hash(x.footer_signing_key_metadata, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{FileMetaData}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{FileMetaData}) + return Thrift.decode(r, FileMetaData) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::FileMetaData) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{FileMetaData}) + Thrift.enter!(r) + f_version = nothing + f_schema = nothing + f_num_rows = nothing + f_row_groups = nothing + f_key_value_metadata = nothing + f_created_by = nothing + f_column_orders = nothing + f_encryption_algorithm = nothing + f_footer_signing_key_metadata = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.I32 + f_version = Thrift.readi32(r) + elseif id == Int16(2) && ty == Thrift.LIST + value_schema = Thrift.readlist(r, SchemaElement) + if value_schema === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_schema = value_schema + end + elseif id == Int16(3) && ty == Thrift.I64 + f_num_rows = Thrift.readi64(r) + elseif id == Int16(4) && ty == Thrift.LIST + value_row_groups = Thrift.readlist(r, RowGroup) + if value_row_groups === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_row_groups = value_row_groups + end + elseif id == Int16(5) && ty == Thrift.LIST + value_key_value_metadata = Thrift.readlist(r, KeyValue) + if value_key_value_metadata === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_key_value_metadata = value_key_value_metadata + end + elseif id == Int16(6) && ty == Thrift.BINARY + f_created_by = Thrift.readstring(r) + elseif id == Int16(7) && ty == Thrift.LIST + value_column_orders = Thrift.readlist(r, ColumnOrder) + if value_column_orders === nothing + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + else + f_column_orders = value_column_orders + end + elseif id == Int16(8) && ty == Thrift.STRUCT + f_encryption_algorithm = Thrift.decode(r, EncryptionAlgorithm) + elseif id == Int16(9) && ty == Thrift.BINARY + f_footer_signing_key_metadata = Thrift.readbinary(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_version === nothing && Thrift.missingfield(:FileMetaData, :version) + f_schema === nothing && Thrift.missingfield(:FileMetaData, :schema) + f_num_rows === nothing && Thrift.missingfield(:FileMetaData, :num_rows) + f_row_groups === nothing && Thrift.missingfield(:FileMetaData, :row_groups) + return FileMetaData(f_version, f_schema, f_num_rows, f_row_groups, f_key_value_metadata, f_created_by, f_column_orders, f_encryption_algorithm, f_footer_signing_key_metadata, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::FileMetaData) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.I32) + Thrift.writei32!(w, x.version) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.LIST) + Thrift.writelist!(w, x.schema) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(3), Thrift.I64) + Thrift.writei64!(w, x.num_rows) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(4), Thrift.LIST) + Thrift.writelist!(w, x.row_groups) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_key_value_metadata = x.key_value_metadata + if value_key_value_metadata !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(5), Thrift.LIST) + Thrift.writelist!(w, value_key_value_metadata) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_created_by = x.created_by + if value_created_by !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(6), Thrift.BINARY) + Thrift.writestring!(w, value_created_by) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_column_orders = x.column_orders + if value_column_orders !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(7), Thrift.LIST) + Thrift.writelist!(w, value_column_orders) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_encryption_algorithm = x.encryption_algorithm + if value_encryption_algorithm !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(8), Thrift.STRUCT) + Thrift.encode!(w, value_encryption_algorithm) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + value_footer_signing_key_metadata = x.footer_signing_key_metadata + if value_footer_signing_key_metadata !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(9), Thrift.BINARY) + Thrift.writebinary!(w, value_footer_signing_key_metadata) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +# Thrift struct FileCryptoMetaData +Base.@kwdef struct FileCryptoMetaData + encryption_algorithm::EncryptionAlgorithm # 1: required EncryptionAlgorithm encryption_algorithm + key_metadata::Union{Nothing, Vector{UInt8}} = nothing # 2: optional binary key_metadata + unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[] +end + +function Base.:(==)(a::FileCryptoMetaData, b::FileCryptoMetaData) + return a.encryption_algorithm == b.encryption_algorithm && a.key_metadata == b.key_metadata && a.unknown_fields == b.unknown_fields +end + +function Base.isequal(a::FileCryptoMetaData, b::FileCryptoMetaData) + return isequal(a.encryption_algorithm, b.encryption_algorithm) && isequal(a.key_metadata, b.key_metadata) && isequal(a.unknown_fields, b.unknown_fields) +end + +function Base.hash(x::FileCryptoMetaData, h::UInt) + h = hash(:FileCryptoMetaData, h) + h = hash(x.encryption_algorithm, h) + h = hash(x.key_metadata, h) + h = hash(x.unknown_fields, h) + return h +end + +function Thrift.typecode(::Core.Type{FileCryptoMetaData}) + return Thrift.STRUCT +end + +function Thrift.readelement(r::Thrift.Reader, ::Core.Type{FileCryptoMetaData}) + return Thrift.decode(r, FileCryptoMetaData) +end + +function Thrift.writeelement!(w::Thrift.Writer, x::FileCryptoMetaData) + Thrift.encode!(w, x) + return +end + +function Thrift.decode(r::Thrift.Reader, ::Core.Type{FileCryptoMetaData}) + Thrift.enter!(r) + f_encryption_algorithm = nothing + f_key_metadata = nothing + unknown = nothing + lastid = Int16(0) + while true + id, ty = Thrift.readfieldheader(r, lastid) + ty == Thrift.STOP && break + lastid = id + if id == Int16(1) && ty == Thrift.STRUCT + f_encryption_algorithm = Thrift.decode(r, EncryptionAlgorithm) + elseif id == Int16(2) && ty == Thrift.BINARY + f_key_metadata = Thrift.readbinary(r) + else + unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty)) + end + end + Thrift.leave!(r) + unknown_fields = Thrift.finishunknown(r, unknown) + f_encryption_algorithm === nothing && Thrift.missingfield(:FileCryptoMetaData, :encryption_algorithm) + return FileCryptoMetaData(f_encryption_algorithm, f_key_metadata, unknown_fields) +end + +function Thrift.encode!(w::Thrift.Writer, x::FileCryptoMetaData) + unknown = x.unknown_fields + lastid = Int16(0) + index = 1 + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + lastid = Thrift.writefieldheader!(w, lastid, Int16(1), Thrift.STRUCT) + Thrift.encode!(w, x.encryption_algorithm) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + value_key_metadata = x.key_metadata + if value_key_metadata !== nothing + lastid = Thrift.writefieldheader!(w, lastid, Int16(2), Thrift.BINARY) + Thrift.writebinary!(w, value_key_metadata) + (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid) + end + Thrift.writeunknownrest!(w, unknown, index, lastid) + Thrift.writestop!(w) + return +end + +end diff --git a/src/nested_reader.jl b/src/nested_reader.jl new file mode 100644 index 0000000..a7c209a --- /dev/null +++ b/src/nested_reader.jl @@ -0,0 +1,826 @@ +# Recursive Dremel assembly from aligned physical leaf streams. + +# The assembly passes recurse one Julia stack frame per plan-tree level, unlike the +# iterative schema parser and plan compiler, so a plan deep enough to be built under +# a raised max_metadata_depth could otherwise overflow the stack here (with corrupted +# process state, not a clean error). Reject such plans up front with a LimitError. +# Keep a wide safety margin below observed process-corrupting stack overflows. Writer +# validation is iterative and can accept deeper synthetic values, but the reader must +# reject them until every assembly pass is iterative. +const _NESTED_READ_MAX_DEPTH = 1024 + +mutable struct _NestedReadState + plan::_NestedPlan + children::Vector{_NestedReadState} + force_required::Bool + occurrences::Int64 + present::Int64 + entries::Int64 + fill_occurrences::Int64 + fill_present::Int64 + fill_entries::Int64 + buffer::Any + validity::Union{Nothing,BitVector} + names::Union{Nothing,Vector{String}} + child_outputs::Union{Nothing,Vector{AbstractVector}} + result::Any +end + +mutable struct _NestedReadContext + streams::Vector{LeafStream} + positions::Vector{Int} + dense_positions::Vector{Int} + logical_values::Vector{AbstractVector} + limits::Limits + fill::Bool +end + +function _nestedreadlabel(plan::_NestedPlan) + path = getfield(plan, :source).path + isempty(path) && return "schema root" + return "nested field $(repr(join(path, ".")))" +end + +function _nestedreadstate(plan::_NestedLeafPlan; force_required::Bool=false) + return _NestedReadState(plan, _NestedReadState[], force_required, + Int64(0), Int64(0), Int64(0), Int64(0), Int64(0), Int64(0), nothing, + nothing, nothing, nothing, nothing) +end + +function _nestedreadstate(plan::_NestedStructPlan; force_required::Bool=false) + children = _NestedReadState[] + sizehint!(children, length(plan.children)) + for child in plan.children + push!(children, _nestedreadstate(child)) + end + return _NestedReadState(plan, children, force_required, Int64(0), Int64(0), + Int64(0), Int64(0), Int64(0), Int64(0), nothing, nothing, nothing, + nothing, nothing) +end + +function _nestedreadstate(plan::_NestedListPlan; force_required::Bool=false) + child = _nestedreadstate(plan.element) + return _NestedReadState(plan, _NestedReadState[child], force_required, + Int64(0), Int64(0), Int64(0), Int64(0), Int64(0), Int64(0), nothing, + nothing, nothing, nothing, nothing) +end + +function _nestedreadstate(plan::_NestedMapPlan; force_required::Bool=false) + count = plan.value === nothing ? 1 : 2 + children = Vector{_NestedReadState}(undef, count) + children[1] = _nestedreadstate(plan.key; force_required=true) + plan.value === nothing || (children[2] = _nestedreadstate(plan.value)) + return _NestedReadState(plan, children, force_required, Int64(0), Int64(0), + Int64(0), Int64(0), Int64(0), Int64(0), nothing, nothing, nothing, + nothing, nothing) +end + +function _nestedreadplancount(plan::_NestedLeafPlan) + return Int64(1) +end + +function _nestedreadplancountsum(count::Int64, child::_NestedPlan) + return try + Base.checked_add(count, _nestedreadplancount(child)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), typemax(Int64))) + end +end + +function _nestedreadplancount(plan::_NestedStructPlan) + count = Int64(1) + for child in plan.children + count = _nestedreadplancountsum(count, child) + end + return count +end + +function _nestedreadplancount(plan::_NestedListPlan) + return _nestedreadplancountsum(Int64(1), plan.element) +end + +function _nestedreadplancount(plan::_NestedMapPlan) + count = _nestedreadplancountsum(Int64(1), plan.key) + plan.value === nothing && return count + return _nestedreadplancountsum(count, plan.value) +end + +function _nestedreadint(value::Integer, label::AbstractString) + value >= 0 || throw(FormatError("negative $label")) + value <= typemax(Int) || throw(FormatError("$label exceeds the Julia index range")) + return Int(value) +end + +function _nestedreadincrement(value::Int64, limits::Limits) + next = try + Base.checked_add(value, Int64(1)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, next, limits.max_container_elements) + return next +end + +function _nestedreadrecord!(context::_NestedReadContext, + state::_NestedReadState, field::Symbol) + if context.fill + fillfield = field === :occurrences ? :fill_occurrences : + field === :present ? :fill_present : :fill_entries + expected = getfield(state, field) + next = getfield(state, fillfield) + Int64(1) + next <= expected || throw(FormatError( + "$(_nestedreadlabel(state.plan)) changed between nested reader passes")) + setfield!(state, fillfield, next) + return next + end + next = _nestedreadincrement(getfield(state, field), context.limits) + setfield!(state, field, next) + return next +end + +function _nestedreadphysicalvalue(value, expected::Type, + leaf::_NestedLeafPlan) + value isa expected || throw(FormatError( + "physical leaf $(repr(join(leaf.source.path, "."))) contains value type " * + "$(typeof(value)); expected $expected")) + if leaf.source.element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY + width = leaf.source.element.type_length + width !== nothing && length(value) == width || throw(FormatError( + "fixed byte-array leaf $(repr(join(leaf.source.path, "."))) contains " * + "a value with the wrong length")) + end + return +end + +function _nestedreadvalidatestream(stream::LeafStream, leaf::_NestedLeafPlan, + rows::Int, limits::Limits) + repetitions = stream.repetition + definitions = stream.definition + length(repetitions) == length(definitions) || throw(FormatError( + "leaf stream repetition and definition counts differ")) + _checklimit(:container_elements, length(repetitions), + limits.max_container_elements) + _checklimit(:container_elements, length(stream.values), + limits.max_container_elements) + maxrepetition = UInt64(leaf.source.max_repetition_level) + maxdefinition = UInt64(leaf.source.max_definition_level) + rowcount = 0 + densecount = 0 + for index in eachindex(repetitions, definitions) + repetition = repetitions[index] + definition = definitions[index] + repetition <= maxrepetition || throw(FormatError( + "repetition level $repetition exceeds the schema maximum $maxrepetition")) + definition <= maxdefinition || throw(FormatError( + "definition level $definition exceeds the schema maximum $maxdefinition")) + repetition <= definition || throw(FormatError( + "repetition level $repetition exceeds definition level $definition")) + iszero(repetition) && (rowcount += 1) + definition == maxdefinition && (densecount += 1) + end + isempty(repetitions) || iszero(first(repetitions)) || throw(FormatError( + "leaf stream starts with a nonzero repetition level")) + rowcount == rows || throw(FormatError( + "leaf stream has $rowcount rows but $rows were expected")) + densecount == length(stream.values) || throw(FormatError( + "leaf stream has $(length(stream.values)) dense values for $densecount present entries")) + expected = _physicaleltype(leaf.source.element.type_) + for value in stream.values + _nestedreadphysicalvalue(value, expected, leaf) + end + return +end + +function _nestedreadvalidatestreams(plan::_NestedSchemaPlan, + streams::AbstractVector, + rows::Int, limits::Limits) + length(streams) == length(plan.leaves) || throw(FormatError( + "nested reader received $(length(streams)) streams for $(length(plan.leaves)) leaves")) + for (index, stream) in enumerate(streams) + stream isa LeafStream || throw(FormatError( + "nested reader stream $index is not a LeafStream")) + _nestedreadvalidatestream(stream, plan.leaves[index], rows, limits) + end + return +end + +function _nestedreadstreams(streams::AbstractVector) + output = Vector{LeafStream}(undef, length(streams)) + for (index, stream) in enumerate(streams) + output[index] = stream + end + return output +end + +function _nestedreadcurrent(context::_NestedReadContext, leafindex::Int) + stream = context.streams[leafindex] + position = context.positions[leafindex] + position <= length(stream) || throw(FormatError( + "leaf $leafindex ends before its sibling occurrence")) + return stream.repetition[position], stream.definition[position] +end + +function _nestedreadstatus(context::_NestedReadContext, plan::_NestedPlan, + startrepetition::UInt64) + range = _nestedleafrange(plan) + isempty(range) && return true + expected = nothing + for rawindex in range + leafindex = Int(rawindex) + repetition, definition = _nestedreadcurrent(context, leafindex) + repetition == startrepetition || throw(FormatError( + "$(_nestedreadlabel(plan)) has misaligned sibling repetition boundaries")) + parent = UInt64(getfield(plan, :parent_definition)) + definition >= parent || throw(FormatError( + "$(_nestedreadlabel(plan)) has a definition below its present ancestor")) + present = definition >= UInt64(getfield(plan, :present_definition)) + if expected === nothing + expected = present + elseif expected != present + throw(FormatError( + "$(_nestedreadlabel(plan)) has inconsistent sibling presence")) + end + end + return something(expected) +end + +function _nestedreadentrystatus(context::_NestedReadContext, + plan::Union{_NestedListPlan,_NestedMapPlan}) + expected = nothing + for rawindex in plan.leaf_range + leafindex = Int(rawindex) + _, definition = _nestedreadcurrent(context, leafindex) + present = definition >= UInt64(plan.entry_definition) + if expected === nothing + expected = present + elseif expected != present + throw(FormatError( + "$(_nestedreadlabel(plan)) has inconsistent sibling entry presence")) + end + end + return something(expected) +end + +function _nestedreadparentrepetition( + plan::Union{_NestedListPlan,_NestedMapPlan}) + plan.repetition_level > 0 || throw(FormatError( + "$(_nestedreadlabel(plan)) has no repeated-entry level")) + return UInt64(plan.repetition_level - Int16(1)) +end + +function _nestedreadconsume!(context::_NestedReadContext, plan::_NestedLeafPlan) + leafindex = Int(first(plan.leaf_range)) + stream = context.streams[leafindex] + position = context.positions[leafindex] + position <= length(stream) || throw(FormatError("leaf $leafindex ends early")) + definition = stream.definition[position] + denseindex = 0 + if definition == UInt64(plan.source.max_definition_level) + denseindex = context.dense_positions[leafindex] + 1 + denseindex <= length(stream.values) || throw(FormatError( + "leaf $leafindex ends before its dense values")) + context.dense_positions[leafindex] = denseindex + end + context.positions[leafindex] = position + 1 + return denseindex +end + +function _nestedreadconsumeplaceholder!(context::_NestedReadContext, + plan::_NestedPlan, leaves::Vector{_NestedLeafPlan}) + for rawindex in _nestedleafrange(plan) + leafindex = Int(rawindex) + stream = context.streams[leafindex] + position = context.positions[leafindex] + position <= length(stream) || throw(FormatError( + "leaf $leafindex ends before its sibling placeholder")) + definition = stream.definition[position] + if definition == UInt64(leaves[leafindex].source.max_definition_level) + denseindex = context.dense_positions[leafindex] + 1 + denseindex <= length(stream.values) || throw(FormatError( + "leaf $leafindex ends before its dense values")) + context.dense_positions[leafindex] = denseindex + end + context.positions[leafindex] = position + 1 + end + return +end + +function _nestedreadboundary(context::_NestedReadContext, + range::UnitRange{Int32}, maximum::UInt64, label::AbstractString) + isempty(range) && return nothing + expected = nothing + ended = nothing + for rawindex in range + leafindex = Int(rawindex) + position = context.positions[leafindex] + stream = context.streams[leafindex] + atend = position > length(stream) + if ended === nothing + ended = atend + elseif ended != atend + throw(FormatError("$label has incomplete sibling streams")) + end + atend && continue + repetition = stream.repetition[position] + repetition <= maximum || throw(FormatError( + "$label leaves an unconsumed repetition level $repetition")) + if expected === nothing + expected = repetition + elseif expected != repetition + throw(FormatError("$label has misaligned next sibling boundaries")) + end + end + something(ended) && return nothing + return something(expected) +end + +function _nestedreadscan!(context::_NestedReadContext, + state::_NestedReadState, startrepetition::UInt64, + leaves::Vector{_NestedLeafPlan}) + return _nestedreadscan!(context, state, state.plan, startrepetition, leaves) +end + +function _nestedreadscan!(context::_NestedReadContext, + state::_NestedReadState, plan::_NestedLeafPlan, startrepetition::UInt64, + leaves::Vector{_NestedLeafPlan}) + outputindex = _nestedreadrecord!(context, state, :occurrences) + present = _nestedreadstatus(context, plan, startrepetition) + denseindex = _nestedreadconsume!(context, plan) + if present + denseindex > 0 || throw(FormatError( + "$(_nestedreadlabel(plan)) is present without a dense value")) + _nestedreadrecord!(context, state, :present) + if context.fill + values = context.logical_values[Int(first(plan.leaf_range))] + state.buffer[Int(outputindex)] = values[denseindex] + end + else + denseindex == 0 || throw(FormatError( + "$(_nestedreadlabel(plan)) is null but has a dense value")) + if context.fill + state.force_required && throw(FormatError( + "$(_nestedreadlabel(plan)) is a null map key")) + state.buffer[Int(outputindex)] = missing + end + end + return present +end + +function _nestedreadscan!(context::_NestedReadContext, + state::_NestedReadState, plan::_NestedStructPlan, startrepetition::UInt64, + leaves::Vector{_NestedLeafPlan}) + outputindex = _nestedreadrecord!(context, state, :occurrences) + present = _nestedreadstatus(context, plan, startrepetition) + boundary = UInt64(plan.source.max_repetition_level) + if !present + _nestedreadconsumeplaceholder!(context, plan, leaves) + _nestedreadboundary(context, plan.leaf_range, boundary, + _nestedreadlabel(plan)) + if context.fill && state.buffer !== nothing + state.buffer[Int(outputindex) + 1] = state.fill_present + end + return false + end + _nestedreadrecord!(context, state, :present) + for child in state.children + _nestedreadscan!(context, child, startrepetition, leaves) + end + _nestedreadboundary(context, plan.leaf_range, boundary, + _nestedreadlabel(plan)) + if context.fill && state.buffer !== nothing + state.buffer[Int(outputindex) + 1] = state.fill_present + end + return true +end + +function _nestedreadscan!(context::_NestedReadContext, + state::_NestedReadState, plan::_NestedListPlan, startrepetition::UInt64, + leaves::Vector{_NestedLeafPlan}) + outputindex = _nestedreadrecord!(context, state, :occurrences) + present = _nestedreadstatus(context, plan, startrepetition) + parentrepetition = _nestedreadparentrepetition(plan) + if !present + _nestedreadconsumeplaceholder!(context, plan, leaves) + _nestedreadboundary(context, plan.leaf_range, parentrepetition, + _nestedreadlabel(plan)) + if context.fill + state.validity === nothing || (state.validity[Int(outputindex)] = false) + state.buffer[Int(outputindex) + 1] = state.fill_entries + end + return false + end + _nestedreadrecord!(context, state, :present) + nonempty = _nestedreadentrystatus(context, plan) + if !nonempty + _nestedreadconsumeplaceholder!(context, plan, leaves) + _nestedreadboundary(context, plan.leaf_range, parentrepetition, + _nestedreadlabel(plan)) + if context.fill + state.validity === nothing || (state.validity[Int(outputindex)] = true) + state.buffer[Int(outputindex) + 1] = state.fill_entries + end + return true + end + itemrepetition = startrepetition + repeated = UInt64(plan.repetition_level) + while true + _nestedreadrecord!(context, state, :entries) + _nestedreadscan!(context, state.children[1], itemrepetition, leaves) + boundary = _nestedreadboundary(context, plan.leaf_range, repeated, + _nestedreadlabel(plan)) + boundary == repeated || break + itemrepetition = repeated + end + boundary = _nestedreadboundary(context, plan.leaf_range, parentrepetition, + _nestedreadlabel(plan)) + if context.fill + state.validity === nothing || (state.validity[Int(outputindex)] = true) + state.buffer[Int(outputindex) + 1] = state.fill_entries + end + boundary === nothing || boundary <= parentrepetition || throw(FormatError( + "$(_nestedreadlabel(plan)) continues after its final item")) + return true +end + +function _nestedreadscan!(context::_NestedReadContext, + state::_NestedReadState, plan::_NestedMapPlan, startrepetition::UInt64, + leaves::Vector{_NestedLeafPlan}) + outputindex = _nestedreadrecord!(context, state, :occurrences) + present = _nestedreadstatus(context, plan, startrepetition) + parentrepetition = _nestedreadparentrepetition(plan) + if !present + _nestedreadconsumeplaceholder!(context, plan, leaves) + _nestedreadboundary(context, plan.leaf_range, parentrepetition, + _nestedreadlabel(plan)) + if context.fill + state.validity === nothing || (state.validity[Int(outputindex)] = false) + state.buffer[Int(outputindex) + 1] = state.fill_entries + end + return false + end + _nestedreadrecord!(context, state, :present) + nonempty = _nestedreadentrystatus(context, plan) + if !nonempty + _nestedreadconsumeplaceholder!(context, plan, leaves) + _nestedreadboundary(context, plan.leaf_range, parentrepetition, + _nestedreadlabel(plan)) + if context.fill + state.validity === nothing || (state.validity[Int(outputindex)] = true) + state.buffer[Int(outputindex) + 1] = state.fill_entries + end + return true + end + itemrepetition = startrepetition + repeated = UInt64(plan.repetition_level) + while true + _nestedreadrecord!(context, state, :entries) + keypresent = _nestedreadscan!(context, state.children[1], + itemrepetition, leaves) + keypresent || throw(FormatError( + "$(_nestedreadlabel(plan)) contains a null map key")) + length(state.children) == 1 || _nestedreadscan!(context, + state.children[2], itemrepetition, leaves) + boundary = _nestedreadboundary(context, plan.leaf_range, repeated, + _nestedreadlabel(plan)) + boundary == repeated || break + itemrepetition = repeated + end + boundary = _nestedreadboundary(context, plan.leaf_range, parentrepetition, + _nestedreadlabel(plan)) + if context.fill + state.validity === nothing || (state.validity[Int(outputindex)] = true) + state.buffer[Int(outputindex) + 1] = state.fill_entries + end + boundary === nothing || boundary <= parentrepetition || throw(FormatError( + "$(_nestedreadlabel(plan)) continues after its final entry")) + return true +end + +function _nestedreadpass!(context::_NestedReadContext, + state::_NestedReadState, plan::_NestedSchemaPlan, rows::Int) + for _ in 1:rows + _nestedreadscan!(context, state, UInt64(0), plan.leaves) + end + boundary = _nestedreadboundary(context, plan.root.leaf_range, UInt64(0), + "schema root") + boundary === nothing || throw(FormatError( + "nested leaf streams contain more rows than the metadata row count")) + for index in eachindex(context.streams) + context.positions[index] == length(context.streams[index]) + 1 || + throw(FormatError("nested reader did not consume every level entry in leaf $index")) + context.dense_positions[index] == length(context.streams[index].values) || + throw(FormatError("nested reader did not consume every dense value in leaf $index")) + end + return +end + +function _nestedreadverifycounts(state::_NestedReadState) + state.present <= state.occurrences || throw(FormatError( + "$(_nestedreadlabel(state.plan)) has more present values than occurrences")) + plan = state.plan + required = state.force_required || + getfield(plan, :parent_definition) == getfield(plan, :present_definition) + required && state.present != state.occurrences && throw(FormatError( + "$(_nestedreadlabel(plan)) is required but has null occurrences")) + if plan isa _NestedStructPlan + iszero(state.entries) || throw(FormatError( + "$(_nestedreadlabel(plan)) records entries for a struct")) + for child in state.children + child.occurrences == state.present || throw(FormatError( + "$(_nestedreadlabel(plan)) has a miscounted struct child")) + end + elseif plan isa _NestedListPlan + length(state.children) == 1 || throw(FormatError( + "$(_nestedreadlabel(plan)) has an invalid reader state")) + state.children[1].occurrences == state.entries || throw(FormatError( + "$(_nestedreadlabel(plan)) has a miscounted list element")) + elseif plan isa _NestedMapPlan + for child in state.children + child.occurrences == state.entries || throw(FormatError( + "$(_nestedreadlabel(plan)) has a miscounted map child")) + end + elseif !iszero(state.entries) + throw(FormatError("$(_nestedreadlabel(plan)) records entries for a leaf")) + end + for child in state.children + _nestedreadverifycounts(child) + end + return +end + +function _nestedreadlogicaltype(plan::_NestedLeafPlan) + physical = _physicaleltype(plan.source.element.type_) + return _logicaleltype(plan.source, physical) +end + +function _nestedreadoutputtype(state::_NestedReadState, + plan::_NestedLeafPlan) + logical = _nestedreadlogicaltype(plan) + required = state.force_required || plan.parent_definition == plan.present_definition + return required ? logical : Union{Missing,logical} +end + +function _nestedreadindexbytes(terminal::Int64, count::Int64) + T = _nestedindextype(terminal) + return _materializedarraybytes(T, count) +end + +function _nestedreadpassbytes(plan::_NestedSchemaPlan) + leaves = Int64(length(plan.leaves)) + nodes = _nestedreadplancount(plan.root) + nodes == plan.plan_count || throw(FormatError( + "nested schema plan count does not match its topology")) + nodes > 0 || throw(FormatError("nested schema plan has no root state")) + edges = nodes - Int64(1) + bytes = _materializedarraybytes(LeafStream, leaves) + bytes = _materializedsum(bytes, _materializedarraybytes(Int, leaves)) + bytes = _materializedsum(bytes, _materializedarraybytes(Int, leaves)) + bytes = _materializedsum(bytes, + _materializedarraybytes(AbstractVector, leaves)) + bytes = _materializedsum(bytes, + _materializedproduct(_nestedreadplusone(nodes), + _MATERIALIZED_OBJECT_BYTES)) + bytes = _materializedsum(bytes, + _materializedproduct(nodes, _MATERIALIZED_ARRAY_HEADER_BYTES)) + bytes = _materializedsum(bytes, + _materializedarraybytes(_NestedReadState, edges; header=false)) + return bytes +end + +function _nestedreadplusone(value::Int64) + return try + Base.checked_add(value, Int64(1)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), typemax(Int64))) + end +end + +function _nestedreadpayloadbytes(plan::_NestedLeafPlan, stream::LeafStream) + _logicalkind(plan.source) === nothing && return Int64(0) + logical = _nestedreadlogicaltype(plan) + isbitstype(logical) && return Int64(0) + bytes = Int64(0) + for value in stream.values + bytes = _materializedsum(bytes, _MATERIALIZED_OBJECT_BYTES) + value isa AbstractVector{UInt8} || continue + bytes = _materializedsum(bytes, length(value)) + end + return bytes +end + +function _nestedreadcharges(state::_NestedReadState, streams::Vector{LeafStream}) + plan = state.plan + if plan isa _NestedLeafPlan + outputtype = _nestedreadoutputtype(state, plan) + final = _materializedarraybytes(outputtype, state.occurrences) + plan.source.element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY && + _logicalkind(plan.source) === nothing && + (final = _materializedsum(final, _MATERIALIZED_OBJECT_BYTES)) + temporary = Int64(0) + if _logicalkind(plan.source) !== nothing + logical = _nestedreadlogicaltype(plan) + temporary = _materializedarraybytes(logical, + length(streams[Int(first(plan.leaf_range))].values)) + final = _materializedsum(final, + _nestedreadpayloadbytes(plan, + streams[Int(first(plan.leaf_range))])) + end + return final, temporary + end + final = _MATERIALIZED_OBJECT_BYTES + temporary = Int64(0) + count = state.occurrences + if plan isa _NestedStructPlan + final = _materializedsum(final, + _materializedarraybytes(String, length(state.children))) + final = _materializedsum(final, + _materializedarraybytes(AbstractVector, length(state.children))) + required = state.force_required || plan.parent_definition == plan.present_definition + required || (final = _materializedsum(final, + _nestedreadindexbytes(state.present, + _nestedreadplusone(count)))) + else + final = _materializedsum(final, + _nestedreadindexbytes(state.entries, + _nestedreadplusone(count))) + required = state.force_required || plan.parent_definition == plan.present_definition + required || (final = _materializedsum(final, + _materializedbitbytes(count))) + end + for child in state.children + childfinal, childtemporary = _nestedreadcharges(child, streams) + final = _materializedsum(final, childfinal) + temporary = _materializedsum(temporary, childtemporary) + end + return final, temporary +end + +function _nestedreadindexarray(terminal::Int64, count::Int64) + length = _nestedreadint(count, "nested index length") + T = _nestedindextype(terminal) + output = Vector{T}(undef, length) + output[1] = zero(T) + return output +end + +function _nestedreadallocate!(state::_NestedReadState) + plan = state.plan + count = _nestedreadint(state.occurrences, + "$(_nestedreadlabel(plan)) occurrence count") + if plan isa _NestedLeafPlan + T = _nestedreadoutputtype(state, plan) + state.buffer = Vector{T}(undef, count) + return + elseif plan isa _NestedStructPlan + required = state.force_required || plan.parent_definition == plan.present_definition + state.buffer = required ? nothing : + _nestedreadindexarray(state.present, + _nestedreadplusone(state.occurrences)) + state.names = Vector{String}(undef, length(plan.children)) + state.child_outputs = Vector{AbstractVector}(undef, length(plan.children)) + for index in eachindex(plan.children) + state.names[index] = plan.children[index].source.element.name + end + else + state.buffer = _nestedreadindexarray(state.entries, + _nestedreadplusone(state.occurrences)) + required = state.force_required || plan.parent_definition == plan.present_definition + state.validity = required ? nothing : falses(count) + end + for child in state.children + _nestedreadallocate!(child) + end + return +end + +function _nestedreadlogicalvalues!(output::Vector{AbstractVector}, + plan::_NestedSchemaPlan, streams::Vector{LeafStream}, limits::Limits) + for index in eachindex(streams) + output[index] = _logicalvalues(plan.leaves[index].source, + streams[index].values; limits=limits) + end + return output +end + +function _nestedreadreset!(state::_NestedReadState) + state.fill_occurrences = 0 + state.fill_present = 0 + state.fill_entries = 0 + for child in state.children + _nestedreadreset!(child) + end + return +end + +function _nestedreadverifyfills(state::_NestedReadState) + state.fill_occurrences == state.occurrences || throw(FormatError( + "$(_nestedreadlabel(state.plan)) occurrence count changed between passes")) + state.fill_present == state.present || throw(FormatError( + "$(_nestedreadlabel(state.plan)) presence count changed between passes")) + state.fill_entries == state.entries || throw(FormatError( + "$(_nestedreadlabel(state.plan)) entry count changed between passes")) + for child in state.children + _nestedreadverifyfills(child) + end + return +end + +function _nestedreadfinalize!(state::_NestedReadState) + for child in state.children + _nestedreadfinalize!(child) + end + plan = state.plan + if plan isa _NestedLeafPlan + if plan.source.element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY && + _logicalkind(plan.source) === nothing + state.result = FixedByteArrayVector{eltype(state.buffer)}( + state.buffer, plan.source.element.type_length) + else + state.result = state.buffer + end + return + elseif plan isa _NestedStructPlan + children = something(state.child_outputs) + for index in eachindex(state.children) + children[index] = state.children[index].result + end + names = something(state.names) + ranks = state.buffer + T = ranks === nothing ? StructValue : Union{Missing,StructValue} + state.result = StructVector{T,typeof(ranks)}( + names, ranks, children, Int(state.occurrences)) + return + elseif plan isa _NestedListPlan + values = state.children[1].result + E = eltype(values) + V = ListValue{E} + T = state.validity === nothing ? V : Union{Missing,V} + state.result = ListVector{T,E,eltype(state.buffer),typeof(state.validity)}( + state.buffer, state.validity, values) + return + end + keys = state.children[1].result + Missing <: eltype(keys) && throw(FormatError( + "$(_nestedreadlabel(plan)) exposes a nullable map key")) + values = length(state.children) == 1 ? nothing : state.children[2].result + K = eltype(keys) + hasvalues = values !== nothing + V = hasvalues ? eltype(values) : Missing + R = MapValue{K,V,hasvalues} + T = state.validity === nothing ? R : Union{Missing,R} + state.result = MapVector{T,K,V,hasvalues,eltype(state.buffer), + typeof(state.validity)}(state.buffer, state.validity, keys, values) + return +end + +function _assemblenested(plan::_NestedSchemaPlan, streams::AbstractVector, + rows::Integer; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + plan.depth <= _NESTED_READ_MAX_DEPTH || throw(LimitError(:nested_read_depth, + Int64(plan.depth), Int64(_NESTED_READ_MAX_DEPTH))) + rowcount = _nestedreadint(rows, "nested row count") + _checklimit(:container_elements, rowcount, limits.max_container_elements) + _nestedreadvalidatestreams(plan, streams, rowcount, limits) + passbytes = _nestedreadpassbytes(plan) + _reserve!(budget, passbytes) + finalreserved = Int64(0) + try + normalized = _nestedreadstreams(streams) + state = _nestedreadstate(plan.root) + positions = ones(Int, length(normalized)) + densepositions = zeros(Int, length(normalized)) + logical = Vector{AbstractVector}(undef, length(normalized)) + context = _NestedReadContext(normalized, positions, densepositions, + logical, limits, false) + _nestedreadpass!(context, state, plan, rowcount) + _nestedreadverifycounts(state) + finalbytes, temporarybytes = _nestedreadcharges(state, normalized) + totalbytes = _materializedsum(finalbytes, temporarybytes) + _reserve!(budget, totalbytes) + finalreserved = totalbytes + _nestedreadlogicalvalues!(logical, plan, normalized, limits) + _nestedreadallocate!(state) + _nestedreadreset!(state) + fill!(positions, 1) + fill!(densepositions, 0) + context.fill = true + _nestedreadpass!(context, state, plan, rowcount) + _nestedreadverifyfills(state) + _nestedreadfinalize!(state) + _release!(budget, _materializedsum(passbytes, temporarybytes)) + return state.result + catch + iszero(finalreserved) || _release!(budget, finalreserved) + _release!(budget, passbytes) + rethrow() + end +end + +function _assemblenested(plan::_NestedSchemaPlan, streams::AbstractVector, + rows::Integer, limits::Limits, budget::_LiveByteBudget) + return _assemblenested(plan, streams, rows; limits=limits, budget=budget) +end diff --git a/src/nested_schema.jl b/src/nested_schema.jl new file mode 100644 index 0000000..3b1638d --- /dev/null +++ b/src/nested_schema.jl @@ -0,0 +1,596 @@ +abstract type _NestedPlan end + +struct _NestedLeafPlan <: _NestedPlan + source::SchemaNode + parent_definition::Int16 + present_definition::Int16 + leaf_range::UnitRange{Int32} +end + +struct _NestedStructPlan <: _NestedPlan + source::SchemaNode + parent_definition::Int16 + present_definition::Int16 + children::Vector{_NestedPlan} + leaf_range::UnitRange{Int32} +end + +struct _NestedListPlan <: _NestedPlan + source::SchemaNode + entry::SchemaNode + parent_definition::Int16 + present_definition::Int16 + entry_definition::Int16 + repetition_level::Int16 + element::_NestedPlan + annotation::Symbol + rule::UInt8 + leaf_range::UnitRange{Int32} +end + +struct _NestedMapPlan <: _NestedPlan + source::SchemaNode + entry::SchemaNode + parent_definition::Int16 + present_definition::Int16 + entry_definition::Int16 + repetition_level::Int16 + key::_NestedPlan + value::Union{Nothing,_NestedPlan} + annotation::Symbol + optional_key::Bool + entry_has_map_key_value::Bool + leaf_range::UnitRange{Int32} +end + +struct _NestedSchemaPlan + source::Schema + root::_NestedStructPlan + leaves::Vector{_NestedLeafPlan} + plan_count::Int64 + depth::Int +end + +mutable struct _NestedPlanBuilder + limits::Limits + budget::_LiveByteBudget + count::Int64 + leaves::Vector{_NestedLeafPlan} + leafmaximum::Int + depth::Int +end + +function _nestedemptyrange() + return Int32(1):Int32(0) +end + +function _nestedleafordinalcount(count::Integer) + count <= typemax(Int32) || throw(LimitError(:container_elements, count, + Int64(typemax(Int32)))) + return Int32(count) +end + +function _nestedleafrange(plan::_NestedPlan) + return getfield(plan, :leaf_range) +end + +function _nestedclaim!(builder::_NestedPlanBuilder) + requested = try + Base.checked_add(builder.count, Int64(1)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + builder.limits.max_container_elements)) + end + _checklimit(:container_elements, requested, + builder.limits.max_container_elements) + _reserveobjects!(builder.budget) + builder.count = requested + return +end + +function _nestedmergeranges(children::Vector{_NestedPlan}) + firstleaf = Int32(0) + lastleaf = Int32(0) + for child in children + range = _nestedleafrange(child) + isempty(range) && continue + if firstleaf == 0 + firstleaf = first(range) + else + first(range) == lastleaf + Int32(1) || + throw(FormatError("nested schema has noncontiguous descendant leaves")) + end + lastleaf = last(range) + end + firstleaf == 0 && return _nestedemptyrange() + return firstleaf:lastleaf +end + +function _nestedmergeranges(left::UnitRange{Int32}, right::UnitRange{Int32}) + isempty(left) && return right + isempty(right) && return left + first(right) == last(left) + Int32(1) || + throw(FormatError("nested schema has noncontiguous descendant leaves")) + return first(left):last(right) +end + +function _nestedmodernannotation(logical::Metadata.LogicalType) + logical.STRING !== nothing && return :STRING + logical.MAP !== nothing && return :MAP + logical.LIST !== nothing && return :LIST + logical.ENUM !== nothing && return :ENUM + logical.DECIMAL !== nothing && return :DECIMAL + logical.DATE !== nothing && return :DATE + logical.TIME !== nothing && return :TIME + logical.TIMESTAMP !== nothing && return :TIMESTAMP + logical.INTEGER !== nothing && return :INTEGER + logical.UNKNOWN !== nothing && return :UNKNOWN + logical.JSON !== nothing && return :JSON + logical.BSON !== nothing && return :BSON + logical.UUID !== nothing && return :UUID + logical.FLOAT16 !== nothing && return :FLOAT16 + logical.VARIANT !== nothing && return :VARIANT + logical.GEOMETRY !== nothing && return :GEOMETRY + logical.GEOGRAPHY !== nothing && return :GEOGRAPHY + isempty(logical.unknown_fields) && return :EMPTY + return :FUTURE +end + +function _nestedknownconverted(converted::Metadata.ConvertedType.T) + return Int32(0) <= converted.value <= Int32(21) +end + +function _nestedrejectcollectionplacement(node::SchemaNode, location::String) + element = node.element + logical = element.logicalType + if logical !== nothing && + (logical.LIST !== nothing || logical.MAP !== nothing) + throw(FormatError("$location $(repr(element.name)) cannot carry a " * + "LIST or MAP logical annotation")) + end + converted = element.converted_type + if converted == Metadata.ConvertedType.LIST || + converted == Metadata.ConvertedType.MAP || + converted == Metadata.ConvertedType.MAP_KEY_VALUE + throw(FormatError("$location $(repr(element.name)) cannot carry converted " * + "LIST, MAP, or MAP_KEY_VALUE metadata")) + end + return +end + +function _nestedvalidategroupmetadata(node::SchemaNode) + element = node.element + logical = element.logicalType + if logical !== nothing + annotation = _nestedmodernannotation(logical) + annotation in (:VARIANT, :FUTURE, :EMPTY) && return + annotation in (:LIST, :MAP) && return + throw(FormatError("logical annotation $annotation on group " * + "$(repr(element.name)) requires a primitive physical type")) + end + converted = element.converted_type + converted === nothing && return + converted in (Metadata.ConvertedType.LIST, Metadata.ConvertedType.MAP, + Metadata.ConvertedType.MAP_KEY_VALUE) && return + _nestedknownconverted(converted) || return + throw(FormatError("converted annotation $(converted.value) on group " * + "$(repr(element.name)) requires a primitive physical type")) +end + +function _nestedgroupkind(node::SchemaNode) + logical = node.element.logicalType + if logical !== nothing + _nestedvalidategroupmetadata(node) + logical.LIST !== nothing && return (:list, :modern_list) + logical.MAP !== nothing && return (:map, :modern_map) + return (:struct, :ordinary) + end + _nestedvalidategroupmetadata(node) + converted = node.element.converted_type + converted == Metadata.ConvertedType.LIST && return (:list, :legacy_list) + converted == Metadata.ConvertedType.MAP && return (:map, :legacy_map) + converted == Metadata.ConvertedType.MAP_KEY_VALUE && + return (:map, :legacy_map_key_value) + return (:struct, :ordinary) +end + +function _nestedthresholds(node::SchemaNode; owned_repeated::Bool=false) + repetition = node.element.repetition_type + if owned_repeated + repetition == Metadata.FieldRepetitionType.REPEATED || + throw(FormatError("nested repeated entry $(repr(node.element.name)) " * + "is not marked REPEATED")) + return (node.max_definition_level, node.max_definition_level) + elseif repetition == Metadata.FieldRepetitionType.REQUIRED + return (node.max_definition_level, node.max_definition_level) + elseif repetition == Metadata.FieldRepetitionType.OPTIONAL + node.max_definition_level > 0 || + throw(FormatError("optional field $(repr(node.element.name)) has an " * + "invalid definition level")) + return (node.max_definition_level - Int16(1), node.max_definition_level) + elseif repetition == Metadata.FieldRepetitionType.REPEATED + throw(FormatError("repeated field $(repr(node.element.name)) was not " * + "normalized as a collection")) + end + throw(FormatError("field $(repr(node.element.name)) has no valid repetition type")) +end + +function _nestedlistrule(outer::SchemaNode, entry::SchemaNode) + entry.element.type_ === nothing || return UInt8(1) + count = length(entry.children) + count == 0 && throw(FormatError("LIST field $(repr(outer.element.name)) has a " * + "zero-field repeated wrapper")) + count >= 2 && return UInt8(2) + onlychild = entry.children[1] + onlychild.element.repetition_type == Metadata.FieldRepetitionType.REPEATED && + return UInt8(3) + entry.element.name == "array" && return UInt8(4) + entry.element.name == string(outer.element.name, "_tuple") && return UInt8(5) + return UInt8(6) +end + +function _nestedmapentrymarker(entry::SchemaNode) + logical = entry.element.logicalType + if logical !== nothing + annotation = _nestedmodernannotation(logical) + annotation in (:FUTURE, :EMPTY) && return false + throw(FormatError("MAP entry group $(repr(entry.element.name)) cannot carry " * + "logical annotation $annotation")) + end + converted = entry.element.converted_type + converted === nothing && return false + converted == Metadata.ConvertedType.MAP_KEY_VALUE && return true + _nestedknownconverted(converted) || return false + throw(FormatError("MAP entry group $(repr(entry.element.name)) cannot carry " * + "converted annotation $(converted.value)")) +end + +function _nestedvalidatemapchild(node::SchemaNode, role::String) + repetition = node.element.repetition_type + repetition == Metadata.FieldRepetitionType.REPEATED && throw(FormatError( + "MAP $role $(repr(node.element.name)) cannot be REPEATED")) + repetition in (Metadata.FieldRepetitionType.REQUIRED, + Metadata.FieldRepetitionType.OPTIONAL) && return + throw(FormatError("MAP $role $(repr(node.element.name)) has no valid repetition type")) +end + +const _NESTED_FRAME_ROOT = UInt8(1) +const _NESTED_FRAME_STRUCT = UInt8(2) +const _NESTED_FRAME_LIST = UInt8(3) +const _NESTED_FRAME_MAP = UInt8(4) +const _NESTED_FRAME_REPEATED = UInt8(5) + +mutable struct _NestedCompileFrame + node::SchemaNode + parent::Union{Nothing,_NestedCompileFrame} + depth::Int + mode::UInt8 + annotation::Symbol + rule::UInt8 + parent_definition::Int16 + present_definition::Int16 + entry::Union{Nothing,SchemaNode} + children::Union{Nothing,Vector{_NestedPlan}} + firstplan::Union{Nothing,_NestedPlan} + secondplan::Union{Nothing,_NestedPlan} + expected::Int + completed::Int + optional_key::Bool + entry_has_map_key_value::Bool +end + +function _nestedframe(builder::_NestedPlanBuilder, node::SchemaNode, parent, + depth::Int, mode::UInt8, annotation::Symbol, rule::UInt8, + parent_definition::Int16, present_definition::Int16, + entry::Union{Nothing,SchemaNode}, + children::Union{Nothing,Vector{_NestedPlan}}, expected::Int; + optional_key::Bool=false, entry_has_map_key_value::Bool=false) + _reserveobjects!(builder.budget) + return _NestedCompileFrame(node, parent, depth, mode, annotation, rule, + parent_definition, present_definition, entry, children, nothing, + nothing, expected, 0, optional_key, entry_has_map_key_value) +end + +function _nestedcheckdepth(builder::_NestedPlanBuilder, depth::Int) + _checklimit(:metadata_depth, depth, builder.limits.max_metadata_depth) + depth > builder.depth && (builder.depth = depth) + return +end + +function _nestedstartroot(builder::_NestedPlanBuilder, node::SchemaNode) + _nestedcheckdepth(builder, 1) + _nestedclaim!(builder) + count = length(node.children) + _checklimit(:container_elements, count, + builder.limits.max_container_elements) + _reservearray!(builder.budget, _NestedPlan, count) + children = _NestedPlan[] + sizehint!(children, count) + return _nestedframe(builder, node, nothing, 1, _NESTED_FRAME_ROOT, + :ordinary, UInt8(0), Int16(0), Int16(0), nothing, children, count) +end + +function _nestedstartleaf(builder::_NestedPlanBuilder, node::SchemaNode, + parentframe, depth::Int, owned_repeated::Bool) + _nestedrejectcollectionplacement(node, "primitive field") + if node.element.repetition_type == Metadata.FieldRepetitionType.REPEATED && + !owned_repeated + node.max_definition_level > 0 || throw(FormatError( + "repeated field $(repr(node.element.name)) has an invalid definition level")) + _nestedcheckdepth(builder, depth) + _nestedclaim!(builder) + definition = node.max_definition_level - Int16(1) + frame = _nestedframe(builder, node, parentframe, depth, + _NESTED_FRAME_REPEATED, :unannotated_repeated, UInt8(0), + definition, definition, node, nothing, 1) + return (nothing, frame) + end + parent, present = _nestedthresholds(node; owned_repeated=owned_repeated) + node.column_index > 0 || throw(FormatError( + "primitive field $(repr(node.element.name)) has no column index")) + length(builder.leaves) < builder.leafmaximum || throw(FormatError( + "nested schema contains more physical leaf occurrences than its raw leaf list")) + _nestedcheckdepth(builder, depth) + _nestedclaim!(builder) + range = node.column_index:node.column_index + plan = _NestedLeafPlan(node, parent, present, range) + push!(builder.leaves, plan) + return (plan, nothing) +end + +function _nestedstartstruct(builder::_NestedPlanBuilder, node::SchemaNode, + parentframe, depth::Int, owned_repeated::Bool) + parent, present = _nestedthresholds(node; owned_repeated=owned_repeated) + _nestedcheckdepth(builder, depth) + _nestedclaim!(builder) + count = length(node.children) + _checklimit(:container_elements, count, + builder.limits.max_container_elements) + _reservearray!(builder.budget, _NestedPlan, count) + children = _NestedPlan[] + sizehint!(children, count) + frame = _nestedframe(builder, node, parentframe, depth, + _NESTED_FRAME_STRUCT, :ordinary, UInt8(0), parent, present, nothing, + children, count) + return (nothing, frame) +end + +function _nestedstartlist(builder::_NestedPlanBuilder, node::SchemaNode, + parentframe, depth::Int, annotation::Symbol, owned_repeated::Bool) + parent, present = _nestedthresholds(node; owned_repeated=owned_repeated) + length(node.children) == 1 || throw(FormatError( + "LIST field $(repr(node.element.name)) must have exactly one child")) + entry = node.children[1] + entry.element.repetition_type == Metadata.FieldRepetitionType.REPEATED || + throw(FormatError("LIST child $(repr(entry.element.name)) must be REPEATED")) + rule = _nestedlistrule(node, entry) + if rule == UInt8(6) + kind, _ = _nestedgroupkind(entry) + kind === :struct || throw(FormatError( + "annotated repeated LIST wrapper $(repr(entry.element.name)) cannot be unwrapped")) + end + _nestedcheckdepth(builder, depth) + _nestedcheckdepth(builder, depth + 1) + _nestedclaim!(builder) + frame = _nestedframe(builder, node, parentframe, depth, _NESTED_FRAME_LIST, + annotation, rule, parent, present, entry, nothing, 1) + return (nothing, frame) +end + +function _nestedstartmap(builder::_NestedPlanBuilder, node::SchemaNode, + parentframe, depth::Int, annotation::Symbol, owned_repeated::Bool) + parent, present = _nestedthresholds(node; owned_repeated=owned_repeated) + length(node.children) == 1 || throw(FormatError( + "MAP field $(repr(node.element.name)) must have exactly one child")) + entry = node.children[1] + entry.element.type_ === nothing || throw(FormatError( + "MAP entry $(repr(entry.element.name)) must be a group")) + entry.element.repetition_type == Metadata.FieldRepetitionType.REPEATED || + throw(FormatError("MAP entry $(repr(entry.element.name)) must be REPEATED")) + count = length(entry.children) + 1 <= count <= 2 || throw(FormatError( + "MAP entry $(repr(entry.element.name)) must have one or two children")) + marker = _nestedmapentrymarker(entry) + keynode = entry.children[1] + _nestedvalidatemapchild(keynode, "key") + valuenode = count == 2 ? entry.children[2] : nothing + valuenode === nothing || _nestedvalidatemapchild(valuenode, "value") + _nestedcheckdepth(builder, depth) + _nestedcheckdepth(builder, depth + 1) + _nestedclaim!(builder) + optionalkey = keynode.element.repetition_type == + Metadata.FieldRepetitionType.OPTIONAL + frame = _nestedframe(builder, node, parentframe, depth, _NESTED_FRAME_MAP, + annotation, UInt8(0), parent, present, entry, nothing, count; + optional_key=optionalkey, entry_has_map_key_value=marker) + return (nothing, frame) +end + +function _nestedstartnode(builder::_NestedPlanBuilder, node::SchemaNode, + parentframe, depth::Int; owned_repeated::Bool=false) + node.element.type_ === nothing || return _nestedstartleaf(builder, node, + parentframe, depth, owned_repeated) + kind, annotation = _nestedgroupkind(node) + repetition = node.element.repetition_type + if kind !== :struct + if repetition == Metadata.FieldRepetitionType.REPEATED && + !owned_repeated + throw(FormatError("annotated collection $(repr(node.element.name)) " * + "cannot be REPEATED outside a parent LIST compatibility form")) + end + kind === :list && return _nestedstartlist(builder, node, parentframe, + depth, annotation, owned_repeated) + return _nestedstartmap(builder, node, parentframe, depth, annotation, + owned_repeated) + end + if repetition == Metadata.FieldRepetitionType.REPEATED && !owned_repeated + node.max_definition_level > 0 || throw(FormatError( + "repeated field $(repr(node.element.name)) has an invalid definition level")) + _nestedcheckdepth(builder, depth) + _nestedclaim!(builder) + definition = node.max_definition_level - Int16(1) + frame = _nestedframe(builder, node, parentframe, depth, + _NESTED_FRAME_REPEATED, :unannotated_repeated, UInt8(0), + definition, definition, node, nothing, 1) + return (nothing, frame) + end + return _nestedstartstruct(builder, node, parentframe, depth, + owned_repeated) +end + +function _nestedframenext(frame::_NestedCompileFrame) + if frame.mode in (_NESTED_FRAME_ROOT, _NESTED_FRAME_STRUCT) + children = frame.children::Vector{_NestedPlan} + node = frame.node.children[frame.completed + 1] + length(children) == frame.completed || throw(AssertionError( + "nested struct frame result count is inconsistent")) + return (node, frame.depth + 1, false) + elseif frame.mode == _NESTED_FRAME_LIST + entry = frame.entry::SchemaNode + if frame.rule == UInt8(6) + return (entry.children[1], frame.depth + 2, false) + end + return (entry, frame.depth + 1, true) + elseif frame.mode == _NESTED_FRAME_MAP + entry = frame.entry::SchemaNode + return (entry.children[frame.completed + 1], frame.depth + 2, false) + end + return (frame.node, frame.depth, true) +end + +function _nestedframeaccept!(frame::_NestedCompileFrame, plan::_NestedPlan) + if frame.mode in (_NESTED_FRAME_ROOT, _NESTED_FRAME_STRUCT) + push!(frame.children::Vector{_NestedPlan}, plan) + elseif frame.completed == 0 + frame.firstplan = plan + else + frame.secondplan = plan + end + frame.completed += 1 + return +end + +function _nestedfinishframe(frame::_NestedCompileFrame) + if frame.mode in (_NESTED_FRAME_ROOT, _NESTED_FRAME_STRUCT) + children = frame.children::Vector{_NestedPlan} + range = _nestedmergeranges(children) + if frame.mode == _NESTED_FRAME_STRUCT && isempty(range) && + frame.present_definition != frame.parent_definition + throw(FormatError("optional leafless group " * + "$(repr(frame.node.element.name)) has no physical leaf that " * + "records its presence")) + end + return _NestedStructPlan(frame.node, frame.parent_definition, + frame.present_definition, children, range) + elseif frame.mode == _NESTED_FRAME_LIST + entry = frame.entry::SchemaNode + element = frame.firstplan::_NestedPlan + range = _nestedleafrange(element) + isempty(range) && throw(FormatError("LIST field " * + "$(repr(frame.node.element.name)) has no physical leaf that " * + "records its entries")) + return _NestedListPlan(frame.node, entry, frame.parent_definition, + frame.present_definition, entry.max_definition_level, + entry.max_repetition_level, element, frame.annotation, frame.rule, + range) + elseif frame.mode == _NESTED_FRAME_MAP + entry = frame.entry::SchemaNode + key = frame.firstplan::_NestedPlan + value = frame.secondplan + range = value === nothing ? _nestedleafrange(key) : + _nestedmergeranges(_nestedleafrange(key), _nestedleafrange(value)) + isempty(range) && throw(FormatError("MAP field " * + "$(repr(frame.node.element.name)) has no physical leaf that " * + "records its entries")) + return _NestedMapPlan(frame.node, entry, frame.parent_definition, + frame.present_definition, entry.max_definition_level, + entry.max_repetition_level, key, value, frame.annotation, + frame.optional_key, frame.entry_has_map_key_value, range) + end + element = frame.firstplan::_NestedPlan + range = _nestedleafrange(element) + isempty(range) && throw(FormatError("repeated field " * + "$(repr(frame.node.element.name)) has no physical leaf that records " * + "its entries")) + return _NestedListPlan(frame.node, frame.node, frame.parent_definition, + frame.present_definition, frame.node.max_definition_level, + frame.node.max_repetition_level, element, frame.annotation, frame.rule, + range) +end + +function _nestedcompileiterative(builder::_NestedPlanBuilder, + rootnode::SchemaNode) + current = _nestedstartroot(builder, rootnode) + pending::Union{Nothing,_NestedPlan} = nothing + while true + if pending !== nothing + _nestedframeaccept!(current, pending) + pending = nothing + end + if current.completed == current.expected + pending = _nestedfinishframe(current) + parent = current.parent + _release!(builder.budget, _MATERIALIZED_OBJECT_BYTES) + parent === nothing && return pending::_NestedStructPlan + current = parent::_NestedCompileFrame + continue + end + node, depth, owned = _nestedframenext(current) + pending, childframe = _nestedstartnode(builder, node, current, depth; + owned_repeated=owned) + childframe === nothing || (current = childframe::_NestedCompileFrame) + end +end + +function _nestedvalidateleafplans(schema::Schema, root::_NestedStructPlan, + leaves::Vector{_NestedLeafPlan}) + length(leaves) == length(schema.leaves) || throw(FormatError( + "nested schema does not cover every physical leaf exactly once")) + for index in eachindex(leaves) + plan = leaves[index] + expected = schema.leaves[index] + plan.source.column_index == index || throw(FormatError( + "nested schema physical leaves are not in column order")) + plan.source.path == expected.path && plan.source.element == expected.element || + throw(FormatError("nested schema leaf plan does not match the raw schema")) + end + leafcount = _nestedleafordinalcount(length(leaves)) + expectedrange = iszero(leafcount) ? _nestedemptyrange() : + Int32(1):leafcount + root.leaf_range == expectedrange || throw(FormatError( + "nested schema root does not have a contiguous physical leaf range")) + return +end + +function _nestedplan(schema::Schema; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + start = _budgetused(budget) + try + rootnode = schema.root + rootnode.element.type_ === nothing || + throw(FormatError("schema root must be a group")) + _nestedrejectcollectionplacement(rootnode, "schema root") + kind, _ = _nestedgroupkind(rootnode) + kind === :struct || throw(FormatError( + "schema root must be an ordinary group")) + _nestedleafordinalcount(length(schema.leaves)) + _reservearray!(budget, _NestedLeafPlan, length(schema.leaves)) + leaves = _NestedLeafPlan[] + sizehint!(leaves, length(schema.leaves)) + _reserveobjects!(budget) + builder = _NestedPlanBuilder(limits, budget, Int64(0), leaves, + length(schema.leaves), 0) + root = _nestedcompileiterative(builder, rootnode) + _nestedvalidateleafplans(schema, root, builder.leaves) + _reserveobjects!(budget) + plan = _NestedSchemaPlan(schema, root, builder.leaves, builder.count, + builder.depth) + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + return plan + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end diff --git a/src/nested_table.jl b/src/nested_table.jl new file mode 100644 index 0000000..ae02a4b --- /dev/null +++ b/src/nested_table.jl @@ -0,0 +1,204 @@ +# Whole-table nested leaf-stream assembly. + +function _validatetablerowgroups(metadata::Metadata.FileMetaData, + schema::Schema, limits::Limits) + declared = metadata.num_rows + declared >= 0 || throw(FormatError( + "file metadata has a negative row count")) + total = Int64(0) + for (rowindex, group) in enumerate(metadata.row_groups) + group.num_rows >= 0 || throw(FormatError( + "row group $rowindex has a negative row count")) + group.num_rows <= typemax(Int) || throw(FormatError( + "row group $rowindex row count overflows")) + length(group.columns) == length(schema.leaves) || throw(FormatError( + "row group $rowindex has $(length(group.columns)) columns for " * + "$(length(schema.leaves)) schema leaves")) + total = try + Base.checked_add(total, group.num_rows) + catch err + err isa OverflowError || rethrow() + throw(FormatError("row group row count overflows Int64")) + end + (iszero(declared) || total <= declared) || throw(FormatError( + "row groups contain more rows than file metadata declares")) + _checklimit(:container_elements, total, limits.max_container_elements) + end + (iszero(declared) || total == declared) || throw(FormatError( + "row groups contain $total rows but file metadata declares $declared")) + total <= typemax(Int) || throw(FormatError( + "table row count exceeds the Julia index range")) + return Int(total) +end + +function _nestedstreamentrycount(metadata::Metadata.FileMetaData, + leaf::_NestedLeafPlan, limits::Limits) + total = Int64(0) + index = Int(leaf.source.column_index) + for group in metadata.row_groups + md = _chunkmetadata(group.columns[index], leaf.source) + total = try + Base.checked_add(total, md.num_values) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, total, + limits.max_container_elements) + end + total <= typemax(Int) || throw(FormatError( + "nested leaf entry count exceeds the Julia index range")) + return Int(total) +end + +function _nestedstreampayloadcharge(::Type{T}, retained::Int64, + count::Int) where {T} + structural = _materializedsum(_leaflevelbytes(count), + _materializedarraybytes(T, count)) + retained >= structural || throw(AssertionError( + "leaf stream retained less than its level and value vectors")) + payload = retained - structural + T === Vector{UInt8} || iszero(payload) || throw(AssertionError( + "fixed-width leaf stream retained an unexpected payload charge")) + return payload +end + +function _appendnestedrowgroup!(repetition::Vector{UInt64}, + definition::Vector{UInt64}, values::Vector{T}, position::Int, + file::File, metadata::Metadata.FileMetaData, schema::Schema, + rowindex::Int, leafindex::Int, count::Int, rows::Int64, + limits::Limits, budget::_LiveByteBudget) where {T} + before = _budgetused(budget) + try + stream = readleafstream(file, metadata, schema, rowindex, leafindex; + expected_rows=rows, limits=limits, budget=budget) + retained = _budgetused(budget) - before + length(stream) == count || throw(FormatError( + "row group $rowindex leaf $leafindex produced $(length(stream)) " * + "of $count declared entries")) + count == 0 || begin + copyto!(repetition, position, stream.repetition, 1, count) + copyto!(definition, position, stream.definition, 1, count) + end + append!(values, stream.values) + payload = _nestedstreampayloadcharge(T, retained, count) + structural = _materializedsum(_leaflevelbytes(count), + _materializedarraybytes(T, count)) + _release!(budget, structural) + return position + count, payload + catch + retained = _budgetused(budget) - before + retained >= 0 || throw(AssertionError( + "leaf stream failure decreased the shared budget")) + iszero(retained) || _release!(budget, retained) + rethrow() + end +end + +function _readnestedstream(file::File, metadata::Metadata.FileMetaData, + schema::Schema, leaf::_NestedLeafPlan, rows::Int, limits::Limits, + budget::_LiveByteBudget, entries::Int) + T = _physicaleltype(leaf.source.element.type_) + repetition = Vector{UInt64}(undef, entries) + definition = Vector{UInt64}(undef, entries) + values = T[] + sizehint!(values, entries) + position = 1 + payloadcharge = Int64(0) + index = Int(leaf.source.column_index) + try + for (rowindex, group) in enumerate(metadata.row_groups) + md = _chunkmetadata(group.columns[index], leaf.source) + count = Int(md.num_values) + position, payload = _appendnestedrowgroup!(repetition, + definition, values, position, file, metadata, schema, + rowindex, index, count, group.num_rows, limits, budget) + payloadcharge = _materializedsum(payloadcharge, payload) + end + position == entries + 1 || throw(AssertionError( + "nested leaf concatenation did not fill its level arrays")) + return LeafStream(repetition, definition, values, + leaf.source.max_repetition_level, + leaf.source.max_definition_level; expected_rows=rows), + payloadcharge + catch + iszero(payloadcharge) || _release!(budget, payloadcharge) + rethrow() + end +end + +function _nestedstreamlayout!(entries::Vector{Int}, + metadata::Metadata.FileMetaData, plan::_NestedSchemaPlan, + limits::Limits) + structural = Int64(0) + for (index, leaf) in enumerate(plan.leaves) + count = _nestedstreamentrycount(metadata, leaf, limits) + entries[index] = count + T = _physicaleltype(leaf.source.element.type_) + charge = _materializedsum(_leaflevelbytes(count), + _materializedarraybytes(T, count)) + structural = _materializedsum(structural, charge) + end + return structural +end + +function _readnestedroot(file::File, metadata::Metadata.FileMetaData, + schema::Schema, plan::_NestedSchemaPlan, limits::Limits, + budget::_LiveByteBudget) + rows = _validatetablerowgroups(metadata, schema, limits) + count = length(plan.leaves) + streamcharge = _materializedsum( + _materializedarraybytes(LeafStream, count), + _materializedproduct(count, _MATERIALIZED_OBJECT_BYTES)) + _reserve!(budget, streamcharge) + streams = LeafStream[] + sizehint!(streams, count) + accountingcharge = _materializedsum(_materializedarraybytes(Int, count), + _materializedarraybytes(Int64, count)) + try + _reserve!(budget, accountingcharge) + catch + _release!(budget, streamcharge) + rethrow() + end + entrycounts = zeros(Int, count) + payloadcharges = zeros(Int64, count) + assembled = false + structuralcharge = Int64(0) + structuralreserved = false + try + structuralcharge = _nestedstreamlayout!(entrycounts, metadata, + plan, limits) + _reserve!(budget, structuralcharge) + structuralreserved = true + for (index, leaf) in enumerate(plan.leaves) + stream, payload = _readnestedstream(file, metadata, schema, + leaf, rows, limits, budget, entrycounts[index]) + push!(streams, stream) + payloadcharges[index] = payload + end + root = _assemblenested(plan, streams, rows; + limits=limits, budget=budget) + assembled = true + for index in eachindex(streams) + if _logicalkind(plan.leaves[index].source) !== nothing + _release!(budget, payloadcharges[index]) + payloadcharges[index] = 0 + end + end + _release!(budget, structuralcharge) + structuralreserved = false + _release!(budget, _materializedsum(streamcharge, accountingcharge)) + return root + catch + if !assembled + for charge in payloadcharges + iszero(charge) || _release!(budget, charge) + end + end + structuralreserved && _release!(budget, structuralcharge) + _release!(budget, _materializedsum(streamcharge, accountingcharge)) + rethrow() + end +end diff --git a/src/page.jl b/src/page.jl new file mode 100644 index 0000000..fb0115b --- /dev/null +++ b/src/page.jl @@ -0,0 +1,210 @@ +# Page framing: PageHeader parsing, payload bounds, and CRC32 verification +# (Parquet 2.13.0 README "Data Pages"/"Column chunks"; PageHeader in parquet.thrift). + +struct PageFrame{B<:AbstractVector{UInt8}} + offset::Int64 + header::Metadata.PageHeader + headerlength::Int + payload::B + materializedcharge::Int64 +end + +function pagekind(header::Metadata.PageHeader) + type = header.type_ + type == Metadata.PageType.DATA_PAGE && return :data_v1 + type == Metadata.PageType.DATA_PAGE_V2 && return :data_v2 + type == Metadata.PageType.DICTIONARY_PAGE && return :dictionary + type == Metadata.PageType.INDEX_PAGE && return :index + return :unknown +end + +function pagekind(frame::PageFrame) + return pagekind(frame.header) +end + +function _subheaderflags(header::Metadata.PageHeader) + return (header.data_page_header !== nothing, + header.index_page_header !== nothing, + header.dictionary_page_header !== nothing, + header.data_page_header_v2 !== nothing) +end + +function _expectedsubheaders(kind::Symbol) + kind === :data_v1 && return (true, false, false, false) + kind === :index && return (false, true, false, false) + kind === :dictionary && return (false, false, true, false) + kind === :data_v2 && return (false, false, false, true) + return nothing +end + +function _validatev2header(header::Metadata.PageHeader) + data = header.data_page_header_v2 + data.num_values >= 0 || throw(FormatError("negative data page V2 value count")) + data.num_nulls >= 0 || throw(FormatError("negative data page V2 null count")) + data.num_rows >= 0 || throw(FormatError("negative data page V2 row count")) + data.num_nulls <= data.num_values || + throw(FormatError("data page V2 null count exceeds its value count")) + data.num_rows <= data.num_values || + throw(FormatError("data page V2 row count exceeds its value count")) + repetition = Int64(data.repetition_levels_byte_length) + definition = Int64(data.definition_levels_byte_length) + repetition >= 0 || throw(FormatError("negative data page V2 repetition-level byte length")) + definition >= 0 || throw(FormatError("negative data page V2 definition-level byte length")) + levels = repetition + definition + levels <= header.compressed_page_size || + throw(FormatError("data page V2 levels exceed its compressed size")) + levels <= header.uncompressed_page_size || + throw(FormatError("data page V2 levels exceed its uncompressed size")) + return +end + +function validatepageheader(header::Metadata.PageHeader) + header.compressed_page_size >= 0 || throw(FormatError("negative compressed page size")) + header.uncompressed_page_size >= 0 || throw(FormatError("negative uncompressed page size")) + kind = pagekind(header) + expected = _expectedsubheaders(kind) + expected === nothing && return kind + _subheaderflags(header) == expected || + throw(FormatError("page header fields do not match the page type $(header.type_)")) + kind === :data_v2 && _validatev2header(header) + return kind +end + +function _pageheaderfailure(err, reader::Thrift.Reader, window::Int64, + available::Int64, limits::Limits) + clipped = window < available + if err isa FormatError && clipped && Thrift.remaining(reader) == 0 + requested = window == typemax(Int64) ? window : window + 1 + throw(LimitError(:page_header_bytes, requested, + limits.max_page_header_bytes)) + end + throw(err) +end + +function _nextpageframecount(count::Int64, limits::Limits) + count >= 0 || throw(ArgumentError( + "physical page frame count must be nonnegative")) + requested = try + Base.checked_add(count, Int64(1)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, requested, + limits.max_container_elements) + return requested +end + +function _pageframeend(offset::Int64, headerlength::Integer, + compressed::Integer) + offset >= 0 || throw(FormatError("negative page frame offset")) + headerlength >= 0 || throw(FormatError( + "negative page header length")) + compressed >= 0 || throw(FormatError( + "negative compressed page size")) + headerlength <= typemax(Int64) && compressed <= typemax(Int64) || + throw(FormatError("page frame end overflows Int64")) + return try + payload = Base.checked_add(offset, Int64(headerlength)) + Base.checked_add(payload, Int64(compressed)) + catch err + err isa OverflowError || rethrow() + throw(FormatError("page frame end overflows Int64")) + end +end + +""" + readpageheader(src, offset, stop, limits) -> (header, headerlength) + +Decode the PageHeader at zero-based `offset`, never reading at or past `stop` and never +decoding more than `limits.max_page_header_bytes`. +""" +function _readpageheader(src::AbstractSource, offset::Int64, stop::Int64, + limits::Limits, budget::_LiveByteBudget) + 0 <= offset < stop || throw(FormatError("page header offset $offset is outside the column chunk")) + total = _checkedsourcelength(src) + stop <= total || throw(FormatError( + "page read stop $stop is past the end of the source")) + available = stop - offset + window = min(Int64(limits.max_page_header_bytes), available) + window > 0 || throw(LimitError(:page_header_bytes, 1, limits.max_page_header_bytes)) + temporary = _reserveobjects!(budget, 2) + bytes = try + _readrangeexact(src, total, offset, window) + catch + _release!(budget, temporary) + rethrow() + end + reader = Thrift.Reader(bytes; limits=limits, budget=budget) + header = try + Thrift.decode(reader, Metadata.PageHeader) + catch err + charge = Thrift.materializedcharge(reader) + _release!(budget, _materializedsum(temporary, charge)) + _pageheaderfailure(err, reader, window, available, limits) + end + charge = Thrift.materializedcharge(reader) + _release!(budget, temporary) + return header, Thrift.consumed(reader), charge, total +end + +function readpageheader(src::AbstractSource, offset::Int64, stop::Int64, + limits::Limits; budget::_LiveByteBudget=_LiveByteBudget(limits)) + header, headerlength, _, _ = _readpageheader(src, offset, stop, limits, + budget) + return header, headerlength +end + +""" + readpage(src, offset, stop, limits) -> PageFrame + +Read one page whose header starts at `offset` and whose bytes must end at or before `stop`. +Both page sizes are charged to `limits.max_page_bytes` before the payload is read, and the +CRC32 of the on-disk payload is verified when the header carries one. +""" +function readpage(src::AbstractSource, offset::Int64, stop::Int64, limits::Limits; + budget::_LiveByteBudget=_LiveByteBudget(limits)) + header, headerlength, headercharge, total = _readpageheader(src, offset, + stop, limits, budget) + framecharge = Int64(0) + try + validatepageheader(header) + compressed = Int64(header.compressed_page_size) + payloadoffset = _pageframeend(offset, headerlength, Int64(0)) + compressed <= stop - payloadoffset || throw(FormatError( + "page payload of $compressed bytes extends past the column chunk end")) + _checklimit(:page_bytes, compressed, limits.max_page_bytes) + _checklimit(:page_bytes, header.uncompressed_page_size, + limits.max_page_bytes) + framecharge = _reserveobjects!(budget, 2) + payload = _readrangeexact(src, total, payloadoffset, compressed) + crc = header.crc + crc === nothing || verifypagechecksum(crc, payload; budget=budget) + charge = _materializedsum(headercharge, framecharge) + return PageFrame(offset, header, headerlength, payload, charge) + catch + _release!(budget, _materializedsum(headercharge, framecharge)) + rethrow() + end +end + +function pageend(frame::PageFrame) + return _pageframeend(frame.offset, frame.headerlength, + frame.header.compressed_page_size) +end + +""" + decompresspage(frame, codec; limits) -> bytes + +Return the page payload decompressed to the exact size declared in its header. +Use this helper only for Data Page V1, dictionary, index, and unknown pages. Data +Page V2 stores repetition and definition levels outside its compressed value +section; use the V2 reader path instead of passing the full V2 payload here. +""" +function decompresspage(frame::PageFrame, codec::Metadata.CompressionCodec.T; + limits::Limits=Limits(), + budget::Union{Nothing,_LiveByteBudget}=nothing) + return decompress(codec, frame.payload, frame.header.uncompressed_page_size; + limits=limits, budget=budget) +end diff --git a/src/page_index.jl b/src/page_index.jl new file mode 100644 index 0000000..78d9666 --- /dev/null +++ b/src/page_index.jl @@ -0,0 +1,868 @@ +# Offset-index construction, serialization, and bounded validation. + +const _OffsetIndexRange = Tuple{Int64,Int64} + +struct _PageIndexInterval + first::Int64 + last::Int64 + index::Bool +end + +struct _PageIndexRangePreflight + groups::Int + intervalcount::Int64 + cumulative::Int64 + overlaps_validated::Bool +end + +function _writevarintsize(value::UInt64) + bytes = Int64(1) + while value >= 0x80 + value >>= 7 + bytes += 1 + end + return bytes +end + +function _writezigzagsize(value::Int32) + encoded = UInt64(reinterpret(UInt32, + xor(value << 1, value >> 31))) + return _writevarintsize(encoded) +end + +function _writezigzagsize(value::Int64) + encoded = reinterpret(UInt64, xor(value << 1, value >> 63)) + return _writevarintsize(encoded) +end + +function _writeoffsetindexencodedsize(index::Metadata.OffsetIndex) + index.unencoded_byte_array_data_bytes === nothing || throw(AssertionError( + "writer OffsetIndex unexpectedly carries byte-array size statistics")) + isempty(index.unknown_fields) || throw(AssertionError( + "writer OffsetIndex unexpectedly carries unknown fields")) + count = length(index.page_locations) + count <= typemax(Int32) || throw(LimitError(:container_elements, + Int64(count), Int64(typemax(Int32)))) + bytes = Int64(2) # field header and struct stop + bytes += 1 # list header + count >= 15 && (bytes = Base.checked_add(bytes, + _writevarintsize(UInt64(count)))) + for location in index.page_locations + isempty(location.unknown_fields) || throw(AssertionError( + "writer PageLocation unexpectedly carries unknown fields")) + bytes = Base.checked_add(bytes, Int64(4)) # three headers and stop + bytes = Base.checked_add(bytes, _writezigzagsize(location.offset)) + bytes = Base.checked_add(bytes, + _writezigzagsize(location.compressed_page_size)) + bytes = Base.checked_add(bytes, + _writezigzagsize(location.first_row_index)) + end + return bytes +end + +function _writeabsoluteoffsetindex(pages::ColumnPages, chunkoffset::Int64, + rows::Int, budget::_LiveByteBudget) + start = _budgetused(budget) + try + rows > 0 || throw(AssertionError( + "writer cannot index a zero-row column chunk")) + isempty(pages.page_locations) && throw(AssertionError( + "writer produced no data-page locations for a nonempty chunk")) + count = length(pages.page_locations) + _reservearray!(budget, Metadata.PageLocation, count) + _reserveobjects!(budget, count + 1) + locations = Metadata.PageLocation[] + sizehint!(locations, count) + previousend = chunkoffset + previousrow = Int64(-1) + chunkend = Base.checked_add(chunkoffset, Int64(length(pages.bytes))) + for (index, relative) in enumerate(pages.page_locations) + relative.offset >= 0 || throw(AssertionError( + "writer produced a negative relative page offset")) + relative.compressed_page_size > 0 || throw(AssertionError( + "writer produced an empty data-page frame")) + absolute = Base.checked_add(chunkoffset, relative.offset) + frameend = Base.checked_add(absolute, + Int64(relative.compressed_page_size)) + absolute >= previousend || throw(AssertionError( + "writer data-page locations overlap or are out of order")) + frameend <= chunkend || throw(AssertionError( + "writer data-page location extends past its column chunk")) + row = relative.first_row_index + validrow = index == 1 ? row == 0 : row > previousrow + validrow || throw(AssertionError( + "writer data-page first-row indexes are not strictly increasing")) + 0 <= row < rows || throw(AssertionError( + "writer data-page first-row index is outside its row group")) + push!(locations, Metadata.PageLocation(offset=absolute, + compressed_page_size=relative.compressed_page_size, + first_row_index=row)) + previousend = frameend + previousrow = row + end + return Metadata.OffsetIndex(page_locations=locations) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writecolumnchunkoffsetindex(chunk::Metadata.ColumnChunk, + offset::Int64, length::Int32) + return Metadata.ColumnChunk( + file_path=chunk.file_path, + file_offset=chunk.file_offset, + meta_data=chunk.meta_data, + offset_index_offset=offset, + offset_index_length=length, + column_index_offset=chunk.column_index_offset, + column_index_length=chunk.column_index_length, + crypto_metadata=chunk.crypto_metadata, + encrypted_column_metadata=chunk.encrypted_column_metadata, + unknown_fields=chunk.unknown_fields, + ) +end + +function _writerowgroupcolumns(group::Metadata.RowGroup, + columns::Vector{Metadata.ColumnChunk}) + return Metadata.RowGroup( + columns=columns, + total_byte_size=group.total_byte_size, + num_rows=group.num_rows, + sorting_columns=group.sorting_columns, + file_offset=group.file_offset, + total_compressed_size=group.total_compressed_size, + ordinal=group.ordinal, + unknown_fields=group.unknown_fields, + ) +end + +function _writeencodeoffsetindex(index::Metadata.OffsetIndex, + cumulative::Int64, limits::Limits, budget::_LiveByteBudget) + exact = _writeoffsetindexencodedsize(index) + exact > 0 || throw(AssertionError("writer produced an empty OffsetIndex")) + exact <= typemax(Int32) || throw(LimitError(:page_index_bytes, + exact, Int64(typemax(Int32)))) + requested = try + Base.checked_add(cumulative, exact) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:page_index_bytes, typemax(Int64), + limits.max_page_index_bytes)) + end + _checklimit(:page_index_bytes, requested, limits.max_page_index_bytes) + charge = _materializedsum(_materializedarraybytes(UInt8, exact), + _MATERIALIZED_OBJECT_BYTES) + _reserve!(budget, charge) + bytes = try + buffer = UInt8[] + sizehint!(buffer, Int(exact)) + writer = Thrift.Writer(buffer) + Thrift.encode!(writer, index) + buffer + catch + _release!(budget, charge) + rethrow() + end + length(bytes) == exact || begin + _release!(budget, charge) + throw(AssertionError( + "writer OffsetIndex size preflight did not match Compact Thrift")) + end + return bytes, charge, requested +end + +function _writeoffsetindexobsoletebytes( + rowgroups::Vector{Metadata.RowGroup}, + indexes::Vector{Vector{Metadata.OffsetIndex}}) + bytes = _materializedarraybytes(Metadata.RowGroup, length(rowgroups)) + bytes = _materializedsum(bytes, + _materializedproduct(length(rowgroups), _MATERIALIZED_OBJECT_BYTES)) + for group in rowgroups + bytes = _materializedsum(bytes, + _materializedarraybytes(Metadata.ColumnChunk, + length(group.columns))) + bytes = _materializedsum(bytes, + _materializedproduct(length(group.columns), + _MATERIALIZED_OBJECT_BYTES)) + end + bytes = _materializedsum(bytes, + _materializedarraybytes(Vector{Metadata.OffsetIndex}, + length(indexes))) + for group in indexes + bytes = _materializedsum(bytes, + _materializedarraybytes(Metadata.OffsetIndex, length(group))) + for index in group + locations = length(index.page_locations) + bytes = _materializedsum(bytes, + _materializedarraybytes(Metadata.PageLocation, locations)) + bytes = _materializedsum(bytes, + _materializedproduct(locations + 1, + _MATERIALIZED_OBJECT_BYTES)) + end + end + return bytes +end + +function _writeoffsetindexsection!(output::Vector{UInt8}, + rowgroups::Vector{Metadata.RowGroup}, + indexes::Vector{Vector{Metadata.OffsetIndex}}, limits::Limits, + budget::_LiveByteBudget) + length(rowgroups) == length(indexes) || throw(AssertionError( + "writer row-group and offset-index counts differ")) + startbudget = _budgetused(budget) + startoutput = length(output) + try + sectioncharge = _reservearray!(budget, UInt8, 0) + section = UInt8[] + _reservearray!(budget, Metadata.RowGroup, length(rowgroups)) + _reserveobjects!(budget, length(rowgroups)) + rebuilt = Metadata.RowGroup[] + sizehint!(rebuilt, length(rowgroups)) + cumulative = Int64(0) + for (group, groupindexes) in zip(rowgroups, indexes) + length(group.columns) == length(groupindexes) || throw( + AssertionError("writer column and OffsetIndex counts differ")) + _reservearray!(budget, Metadata.ColumnChunk, + length(group.columns)) + _reserveobjects!(budget, length(group.columns)) + columns = Metadata.ColumnChunk[] + sizehint!(columns, length(group.columns)) + for (chunk, index) in zip(group.columns, groupindexes) + isempty(index.page_locations) && throw(AssertionError( + "writer cannot serialize an empty OffsetIndex")) + offset = Base.checked_add(Int64(startoutput), + Int64(length(section))) + bytes, charge, cumulative = _writeencodeoffsetindex(index, + cumulative, limits, budget) + try + growth = _reserveoutputgrowth!(budget, length(bytes)) + sectioncharge = _materializedsum(sectioncharge, growth) + append!(section, bytes) + finally + _release!(budget, charge) + end + push!(columns, _writecolumnchunkoffsetindex(chunk, offset, + Int32(length(bytes)))) + end + push!(rebuilt, _writerowgroupcolumns(group, columns)) + end + _reserveoutputgrowth!(budget, length(section)) + append!(output, section) + section = nothing + _release!(budget, sectioncharge) + return rebuilt + catch + resize!(output, startoutput) + used = _budgetused(budget) + used > startbudget && _release!(budget, used - startbudget) + rethrow() + end +end + +function _offsetindexcumulative(current::Int64, length::Int64, + limits::Limits) + requested = try + Base.checked_add(current, length) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:page_index_bytes, typemax(Int64), + limits.max_page_index_bytes)) + end + _checklimit(:page_index_bytes, requested, limits.max_page_index_bytes) + return requested +end + +function _pageindexrangeend(offset::Int64, length::Int64, + message::String) + return try + Base.checked_add(offset, length) + catch err + err isa OverflowError || rethrow() + throw(FormatError(message)) + end +end + +function _pageindexintervalcount(current::Int64, additional::Int64, + limits::Limits) + current >= 0 && additional >= 0 || throw(AssertionError( + "page-index interval counts must be nonnegative")) + requested = try + Base.checked_add(current, additional) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, requested, + limits.max_container_elements) + return requested +end + +function _offsetindexrange(file::File, chunk::Metadata.ColumnChunk, + cumulative::Int64, limits::Limits) + offset = chunk.offset_index_offset + length = chunk.offset_index_length + columnoffset = chunk.column_index_offset + columnlength = chunk.column_index_length + (columnoffset === nothing) == (columnlength === nothing) || + throw(FormatError( + "column-index offset and length must be present together")) + (offset === nothing) == (length === nothing) || throw(FormatError( + "column chunk offset-index offset and length must be present together")) + columnoffset !== nothing && offset === nothing && throw(FormatError( + "column index is present without its required offset index")) + offset === nothing && return nothing, cumulative + offset >= 4 || throw(FormatError( + "offset-index offset $offset is inside the file header")) + length > 0 || throw(FormatError( + "offset-index length must be positive, got $length")) + stop = _pageindexrangeend(offset, Int64(length), + "offset-index range overflows Int64") + stop <= file.footer.offset || throw(FormatError( + "offset-index range extends past the footer")) + cumulative = _offsetindexcumulative(cumulative, Int64(length), limits) + return (offset, Int64(length)), cumulative +end + +function _columnindexrange(file::File, chunk::Metadata.ColumnChunk) + offset = chunk.column_index_offset + length = chunk.column_index_length + (offset === nothing) == (length === nothing) || throw(FormatError( + "column-index offset and length must be present together")) + offset === nothing && return nothing + chunk.offset_index_offset === nothing && throw(FormatError( + "column index is present without its required offset index")) + offset >= 4 || throw(FormatError( + "column-index offset $offset is inside the file header")) + length > 0 || throw(FormatError( + "column-index length must be positive, got $length")) + stop = _pageindexrangeend(offset, Int64(length), + "column-index range overflows Int64") + stop <= file.footer.offset || throw(FormatError( + "column-index range extends past the footer")) + return (Int64(offset), Int64(length)) +end + +function _decodeoffsetindex(file::File, offset::Int64, length::Int64, + limits::Limits, budget::_LiveByteBudget) + start = _budgetused(budget) + temporary = Int64(0) + try + temporary = _reserveobjects!(budget, 2) + total = _checkedsourcelength(file.source) + bytes = _readrangeexact(file.source, total, offset, length) + reader = Thrift.Reader(bytes; limits=limits, budget=budget) + index = Thrift.decode(reader, Metadata.OffsetIndex) + Thrift.remaining(reader) == 0 || throw(FormatError( + "offset index has trailing Compact Thrift bytes")) + _release!(budget, temporary) + return index + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _validatepageindexintervals!(intervals::Vector{_PageIndexInterval}) + sort!(intervals; by=interval -> (interval.first, interval.last, + interval.index)) + previousend = Int64(-1) + for interval in intervals + interval.last > interval.first || throw(AssertionError( + "page-index preflight retained an empty interval")) + interval.first >= previousend || throw(FormatError( + "physical column, page-index, or bloom-filter storage ranges overlap")) + previousend = interval.last + end + return +end + +function _appendpageindexintervals!( + intervals::Union{Nothing,Vector{_PageIndexInterval}}, file::File, + chunk::Metadata.ColumnChunk, node::SchemaNode) + md = _chunkmetadata(chunk, node) + physicalstart, physicalstop = _chunkrange(md, file.footer.offset) + range = chunk.offset_index_offset === nothing ? nothing : + (Int64(chunk.offset_index_offset), Int64(chunk.offset_index_length)) + columnrange = chunk.column_index_offset === nothing ? nothing : + (Int64(chunk.column_index_offset), Int64(chunk.column_index_length)) + bloomrange = _bloomfilterrange(md, file.footer.offset) + intervals === nothing && return range + physicalstop > physicalstart && push!(intervals, + _PageIndexInterval(physicalstart, physicalstop, false)) + for (declared, message) in ((range, "offset-index range overflows Int64"), + (columnrange, "column-index range overflows Int64"), + (bloomrange, "bloom-filter range overflows Int64")) + declared === nothing && continue + first, length = declared + last = _pageindexrangeend(first, length, message) + push!(intervals, _PageIndexInterval(first, last, true)) + end + return range +end + +function _preflightoffsetindexdeclarations(file::File, + metadata::Metadata.FileMetaData, schema::Schema, limits::Limits) + groups = length(metadata.row_groups) + totalchunks = Int64(0) + intervalcount = Int64(0) + cumulative = Int64(0) + for (groupindex, group) in enumerate(metadata.row_groups) + length(group.columns) == length(schema.leaves) || throw(FormatError( + "row group $groupindex column count does not match the schema")) + totalchunks = try + Base.checked_add(totalchunks, Int64(length(group.columns))) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, totalchunks, + limits.max_container_elements) + for (chunk, node) in zip(group.columns, schema.leaves) + md = _chunkmetadata(chunk, node) + physicalstart, physicalstop = _chunkrange(md, + file.footer.offset) + range, cumulative = _offsetindexrange(file, chunk, cumulative, + limits) + columnrange = _columnindexrange(file, chunk) + bloomrange = _bloomfilterrange(md, file.footer.offset) + additional = Int64(physicalstop > physicalstart) + + Int64(range !== nothing) + Int64(columnrange !== nothing) + + Int64(bloomrange !== nothing) + intervalcount = _pageindexintervalcount(intervalcount, + additional, limits) + end + end + return _PageIndexRangePreflight(groups, intervalcount, cumulative, false) +end + +function _validatepageindexdeclarationoverlaps!(file::File, + metadata::Metadata.FileMetaData, schema::Schema, + preflight::_PageIndexRangePreflight, budget::_LiveByteBudget) + preflight.overlaps_validated && return preflight + intervalcharge = Int64(0) + try + intervalcharge = _reservearray!(budget, _PageIndexInterval, + preflight.intervalcount) + intervals = _PageIndexInterval[] + sizehint!(intervals, Int(preflight.intervalcount)) + for group in metadata.row_groups + for (chunk, node) in zip(group.columns, schema.leaves) + _appendpageindexintervals!(intervals, file, chunk, node) + end + end + length(intervals) == preflight.intervalcount || throw(AssertionError( + "page-index declaration count changed during preflight")) + _validatepageindexintervals!(intervals) + finally + iszero(intervalcharge) || _release!(budget, intervalcharge) + end + return _PageIndexRangePreflight(preflight.groups, + preflight.intervalcount, preflight.cumulative, true) +end + +function _preflightoffsetindexranges(file::File, + metadata::Metadata.FileMetaData, schema::Schema, limits::Limits, + budget::_LiveByteBudget, preflight::_PageIndexRangePreflight) + groups = preflight.groups + intervalcount = preflight.intervalcount + cumulative = preflight.cumulative + rangecharge = Int64(0) + intervalcharge = Int64(0) + try + rangecharge = _reservearray!(budget, + Vector{Union{Nothing,_OffsetIndexRange}}, groups) + if !preflight.overlaps_validated + intervalcharge = _reservearray!(budget, _PageIndexInterval, + intervalcount) + end + intervals = preflight.overlaps_validated ? nothing : + _PageIndexInterval[] + intervals === nothing || sizehint!(intervals, Int(intervalcount)) + ranges = Vector{Union{Nothing,_OffsetIndexRange}}[] + sizehint!(ranges, groups) + for group in metadata.row_groups + innercharge = _reservearray!(budget, + Union{Nothing,_OffsetIndexRange}, length(group.columns)) + rangecharge = _materializedsum(rangecharge, innercharge) + groupranges = Union{Nothing,_OffsetIndexRange}[] + sizehint!(groupranges, length(group.columns)) + for (chunk, node) in zip(group.columns, schema.leaves) + range = _appendpageindexintervals!(intervals, file, chunk, + node) + push!(groupranges, range) + end + push!(ranges, groupranges) + end + if intervals !== nothing + length(intervals) == intervalcount || throw(AssertionError( + "page-index declaration count changed during materialization")) + _validatepageindexintervals!(intervals) + end + _release!(budget, intervalcharge) + intervalcharge = Int64(0) + return ranges, cumulative, rangecharge + catch + iszero(intervalcharge) || _release!(budget, intervalcharge) + iszero(rangecharge) || _release!(budget, rangecharge) + rethrow() + end +end + +function _preflightoffsetindexranges(file::File, + metadata::Metadata.FileMetaData, schema::Schema, limits::Limits, + budget::_LiveByteBudget) + preflight = _preflightoffsetindexdeclarations(file, metadata, schema, + limits) + return _preflightoffsetindexranges(file, metadata, schema, limits, + budget, preflight) +end + +function _offsetindexpagerows(frame::PageFrame, + md::Metadata.ColumnMetaData, node::SchemaNode, limits::Limits, + budget::_LiveByteBudget) + kind = pagekind(frame) + if kind === :data_v2 + header = frame.header.data_page_header_v2 + rows = Int64(header.num_rows) + rows > 0 || throw(FormatError( + "data page V2 has no rows and cannot be indexed")) + entries = Int(header.num_values) + if node.max_repetition_level == 0 + rows == entries || throw(FormatError( + "flat data page V2 row and value counts differ")) + return rows + end + entries > 0 || throw(FormatError( + "nested data page V2 has no values")) + _checklimit(:container_elements, entries, + limits.max_container_elements) + repetitionbytes = Int(header.repetition_levels_byte_length) + repetitionbytes <= length(frame.payload) || throw(FormatError( + "data page V2 repetition levels extend past its payload")) + working = _materializedsum( + _materializedarraybytes(UInt64, entries), + _materializedproduct(2, _MATERIALIZED_OBJECT_BYTES)) + _reserve!(budget, working) + try + bytes = @view frame.payload[1:repetitionbytes] + repetition = _decodelevelsv2(bytes, entries, + Int(node.max_repetition_level), "repetition", limits) + iszero(first(repetition)) || throw(FormatError( + "nested data page V2 does not begin at a row boundary")) + actual = Int64(Base.count(iszero, repetition)) + actual == rows || throw(FormatError( + "data page V2 repetition levels do not match num_rows")) + finally + _release!(budget, working) + end + return rows + end + kind === :data_v1 || throw(AssertionError( + "row counting requires a data page")) + node.max_repetition_level == 0 && begin + count = Int64(frame.header.data_page_header.num_values) + count > 0 || throw(FormatError( + "data page V1 has no values and cannot be indexed")) + return count + end + header = frame.header.data_page_header + count = Int(header.num_values) + count > 0 || throw(FormatError( + "data page V1 has no values and cannot begin a row")) + _checklimit(:container_elements, count, limits.max_container_elements) + uncompressed = Int64(frame.header.uncompressed_page_size) + working = _materializedsum( + _materializedarraybytes(UInt8, uncompressed), + _materializedarraybytes(UInt64, count)) + working = _materializedsum(working, + _materializedproduct(3, _MATERIALIZED_OBJECT_BYTES)) + _reserve!(budget, working) + try + bytes = decompresspage(frame, md.codec; limits=limits, budget=budget) + repetition, _ = _decodelevelv1(bytes, count, + header.repetition_level_encoding, + Int(node.max_repetition_level), 1, "repetition", limits) + iszero(first(repetition)) || throw(FormatError( + "nested data page V1 does not begin at a row boundary")) + rows = Int64(Base.count(iszero, repetition)) + rows > 0 || throw(FormatError( + "nested data page V1 has no row boundary")) + return rows + finally + _release!(budget, working) + end +end + +function _offsetindexpageentries(frame::PageFrame, node::SchemaNode) + kind = pagekind(frame) + if kind === :data_v2 + header = frame.header.data_page_header_v2 + rows = Int64(header.num_rows) + rows > 0 || throw(FormatError( + "data page V2 has no rows and cannot be indexed")) + entries = Int64(header.num_values) + if node.max_repetition_level == 0 + rows == entries || throw(FormatError( + "flat data page V2 row and value counts differ")) + else + entries > 0 || throw(FormatError( + "nested data page V2 has no values")) + end + return entries + end + kind === :data_v1 || throw(AssertionError( + "entry counting requires a data page")) + entries = Int64(frame.header.data_page_header.num_values) + entries > 0 || throw(FormatError( + "data page V1 has no values and cannot be indexed")) + return entries +end + +function _validateoffsetindexsizes(index::Metadata.OffsetIndex, + node::SchemaNode) + sizes = index.unencoded_byte_array_data_bytes + sizes === nothing && return + node.element.type_ == Metadata.Type.BYTE_ARRAY || throw(FormatError( + "offset-index byte-array sizes require a BYTE_ARRAY column")) + length(sizes) == length(index.page_locations) || throw(FormatError( + "offset-index byte-array size count does not match its page locations")) + all(value -> value >= 0, sizes) || throw(FormatError( + "offset-index byte-array sizes must be nonnegative")) + return +end + +function _validateoffsetindexframes(file::File, + chunk::Metadata.ColumnChunk, node::SchemaNode, rows::Int64, + index::Metadata.OffsetIndex, limits::Limits, + budget::_LiveByteBudget) + rows >= 0 || throw(FormatError( + "offset index belongs to a negative-row row group")) + md = _chunkmetadata(chunk, node) + start, stop = _chunkrange(md, file.footer.offset) + locations = index.page_locations + _validateoffsetindexsizes(index, node) + row = Int64(0) + values = Int64(0) + locationindex = 1 + position = start + dictionaryseen = false + dataseen = false + indexseen = false + framecount = Int64(0) + while position < stop + frame = readpage(file.source, position, stop, limits; budget=budget) + try + frameend = pageend(frame) + frameend > position || throw(FormatError( + "column chunk contains a nonadvancing page frame")) + frameend <= stop || throw(FormatError( + "page frame extends past the column chunk")) + kind = pagekind(frame) + dictionaryseen, dataseen, indexseen = _chunkpageoffsetstate(md, + position, kind, dictionaryseen, dataseen, indexseen) + _validatedpageentrycount(frame, limits) + if kind === :data_v1 || kind === :data_v2 + locationindex <= length(locations) || throw(FormatError( + "offset index omits a data page")) + location = locations[locationindex] + location.offset == position || throw(FormatError( + "offset-index page offset does not match its physical frame")) + location.compressed_page_size > 0 || throw(FormatError( + "offset-index page frame size must be positive")) + Int64(location.compressed_page_size) == frameend - position || + throw(FormatError( + "offset-index page size does not match its physical frame")) + location.first_row_index == row || throw(FormatError( + "offset-index first-row value does not match its data pages")) + pagevalues = _offsetindexpageentries(frame, node) + nextvalues = try + Base.checked_add(values, pagevalues) + catch err + err isa OverflowError || rethrow() + throw(FormatError( + "offset-index value count overflows Int64")) + end + nextvalues <= md.num_values || throw(FormatError( + "offset-index data pages exceed the column value count")) + framecount = _nextpageframecount(framecount, limits) + pagerows = _offsetindexpagerows(frame, md, node, limits, + budget) + values = nextvalues + row = try + Base.checked_add(row, pagerows) + catch err + err isa OverflowError || rethrow() + throw(FormatError("offset-index row count overflows Int64")) + end + row <= rows || throw(FormatError( + "offset-index data pages exceed the row-group row count")) + locationindex += 1 + else + framecount = _nextpageframecount(framecount, limits) + end + position = frameend + finally + _release!(budget, frame.materializedcharge) + end + end + position == stop || throw(FormatError( + "column chunk page walk does not end at its declared boundary")) + _validatechunkpageoffsets(md, dictionaryseen, dataseen, indexseen) + locationindex == length(locations) + 1 || throw(FormatError( + "offset index contains a location with no physical data page")) + row == rows || throw(FormatError( + "offset-index data-page rows do not match the row group")) + values == md.num_values || throw(FormatError( + "offset-index data-page values do not match the column chunk")) + return +end + +function _readoffsetindexpreflighted(file::File, + chunk::Metadata.ColumnChunk, + node::SchemaNode, rows::Int64, + range::Union{Nothing,_OffsetIndexRange}, limits::Limits, + budget::_LiveByteBudget) + range === nothing && return nothing + start = _budgetused(budget) + try + offset, length = range + index = _decodeoffsetindex(file, offset, length, limits, budget) + _validateoffsetindexframes(file, chunk, node, rows, index, limits, + budget) + return index + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _rangesoverlap(left::_OffsetIndexRange, + right::_OffsetIndexRange) + left[1] >= 0 && right[1] >= 0 || throw(FormatError( + "page-index interval offset must be nonnegative")) + left[2] >= 0 && right[2] >= 0 || throw(FormatError( + "page-index interval length must be nonnegative")) + leftstop = _pageindexrangeend(left[1], left[2], + "page-index interval range overflows Int64") + rightstop = _pageindexrangeend(right[1], right[2], + "page-index interval range overflows Int64") + return left[1] < rightstop && right[1] < leftstop +end + +function _readoffsetindex(file::File, chunk::Metadata.ColumnChunk, + node::SchemaNode, rows::Int64, cumulative::Int64, limits::Limits, + budget::_LiveByteBudget) + start = _budgetused(budget) + try + range, cumulative = _offsetindexrange(file, chunk, cumulative, + limits) + columnrange = _columnindexrange(file, chunk) + range === nothing && return nothing, cumulative + md = _chunkmetadata(chunk, node) + physicalstart, physicalstop = _chunkrange(md, file.footer.offset) + physical = (physicalstart, physicalstop - physicalstart) + iszero(physical[2]) || !_rangesoverlap(range, physical) || + throw(FormatError( + "offset-index range overlaps its physical column chunk")) + columnrange === nothing || !_rangesoverlap(range, columnrange) || + throw(FormatError( + "column-index and offset-index ranges overlap")) + if columnrange !== nothing && !iszero(physical[2]) + !_rangesoverlap(columnrange, physical) || throw(FormatError( + "column-index range overlaps its physical column chunk")) + end + rawcharge = _reservearray!(budget, UInt8, range[2]) + index = _readoffsetindexpreflighted(file, chunk, node, rows, + range, limits, budget) + _release!(budget, rawcharge) + return index, cumulative + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _readoffsetindex(file::File, chunk::Metadata.ColumnChunk, + node::SchemaNode, rows::Int64, limits::Limits, + budget::_LiveByteBudget) + index, _ = _readoffsetindex(file, chunk, node, rows, Int64(0), limits, + budget) + return index +end + +function _readoffsetindexes(file::File, + metadata::Metadata.FileMetaData, schema::Schema; + limits::Limits=Limits(), budget::_LiveByteBudget=_LiveByteBudget(limits), + preflight::Union{Nothing,_PageIndexRangePreflight}=nothing) + start = _budgetused(budget) + try + length(metadata.row_groups) <= limits.max_container_elements || + throw(LimitError(:container_elements, + Int64(length(metadata.row_groups)), + limits.max_container_elements)) + _reservearray!(budget, + Vector{Union{Nothing,Metadata.OffsetIndex}}, + length(metadata.row_groups)) + output = Vector{Union{Nothing,Metadata.OffsetIndex}}[] + sizehint!(output, length(metadata.row_groups)) + ranges, cumulative, rangecharge = if preflight === nothing + _preflightoffsetindexranges(file, metadata, schema, limits, budget) + else + _preflightoffsetindexranges(file, metadata, schema, limits, budget, + preflight) + end + indexcharge = iszero(cumulative) ? Int64(0) : + _reservearray!(budget, UInt8, cumulative) + for (groupindex, (group, groupranges)) in enumerate(zip( + metadata.row_groups, ranges)) + group.num_rows >= 0 || throw(FormatError( + "row group $groupindex has a negative row count")) + _reservearray!(budget, Union{Nothing,Metadata.OffsetIndex}, + length(group.columns)) + indexes = Union{Nothing,Metadata.OffsetIndex}[] + sizehint!(indexes, length(group.columns)) + for (chunk, node, range) in zip(group.columns, schema.leaves, + groupranges) + index = _readoffsetindexpreflighted(file, chunk, node, + group.num_rows, range, limits, budget) + push!(indexes, index) + end + push!(output, indexes) + end + _release!(budget, _materializedsum(rangecharge, indexcharge)) + return output + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _readoffsetindexes(file::File, + metadata::Metadata.FileMetaData, schema::Schema, limits::Limits, + budget::_LiveByteBudget) + return _readoffsetindexes(file, metadata, schema; limits=limits, + budget=budget) +end + +function _validateoffsetindexes!(file::File, + metadata::Metadata.FileMetaData, schema::Schema, limits::Limits, + budget::_LiveByteBudget, + preflight::Union{Nothing,_PageIndexRangePreflight}=nothing) + start = _budgetused(budget) + try + _readoffsetindexes(file, metadata, schema; limits=limits, + budget=budget, preflight=preflight) + finally + used = _budgetused(budget) + used > start && _release!(budget, used - start) + end + return +end diff --git a/src/plain.jl b/src/plain.jl new file mode 100644 index 0000000..6161065 --- /dev/null +++ b/src/plain.jl @@ -0,0 +1,146 @@ +function _requirebytes(bytes::AbstractVector{UInt8}, offset::Integer, count::Integer) + offset >= 1 || throw(BoundsError(bytes, offset)) + count >= 0 || throw(ArgumentError("byte count must be nonnegative")) + last = Base.checked_add(Int(offset) - 1, Int(count)) + last <= length(bytes) || throw(FormatError("truncated PLAIN value")) + return +end + +function _readlittle(::Type{U}, bytes::AbstractVector{UInt8}, offset::Int) where {U<:Unsigned} + width = sizeof(U) + _requirebytes(bytes, offset, width) + value = zero(U) + @inbounds for index in 0:(width - 1) + value |= U(bytes[offset + index]) << (8 * index) + end + return value, offset + width +end + +function _writelittle!(output::Vector{UInt8}, value::U) where {U<:Unsigned} + @inbounds for index in 0:(sizeof(U) - 1) + push!(output, UInt8((value >> (8 * index)) & U(0xff))) + end + return +end + +function _plainbits(::Type{Int32}) + return UInt32 +end + +function _plainbits(::Type{Int64}) + return UInt64 +end + +function _plainbits(::Type{Float32}) + return UInt32 +end + +function _plainbits(::Type{Float64}) + return UInt64 +end + +function _fromplainbits(::Type{T}, value::U) where {T,U<:Unsigned} + return reinterpret(T, value) +end + +function _toplainbits(::Type{U}, value::T) where {U<:Unsigned,T} + return reinterpret(U, value) +end + +function decode_plain(::Type{Bool}, bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits()) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + bytecount = cld(Int(count), 8) + position = Int(offset) + _requirebytes(bytes, position, bytecount) + output = Vector{Bool}(undef, Int(count)) + @inbounds for index in 0:(Int(count) - 1) + output[index + 1] = !iszero(bytes[position + (index >> 3)] & (UInt8(1) << (index & 7))) + end + return output, position + bytecount +end + +function decode_plain(::Type{T}, bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits()) where {T<:Union{Int32,Int64,Float32,Float64}} + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + position = Int(offset) + total = Base.checked_mul(Int(count), sizeof(T)) + _requirebytes(bytes, position, total) + output = Vector{T}(undef, Int(count)) + U = _plainbits(T) + @inbounds for index in eachindex(output) + bits, position = _readlittle(U, bytes, position) + output[index] = _fromplainbits(T, bits) + end + return output, position +end + +function decode_plain_byte_array(bytes::AbstractVector{UInt8}, count::Integer; + offset::Integer=1, limits::Limits=Limits()) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + position = Int(offset) + _requirebytes(bytes, position, Base.checked_mul(Int(count), 4)) + output = Vector{Vector{UInt8}}(undef, Int(count)) + for index in eachindex(output) + rawlength, position = _readlittle(UInt32, bytes, position) + length = reinterpret(Int32, rawlength) + length >= 0 || throw(FormatError("negative PLAIN byte-array length")) + _checklimit(:string_bytes, length, limits.max_string_bytes) + _requirebytes(bytes, position, length) + output[index] = collect(@view bytes[position:(position + length - 1)]) + position += length + end + return output, position +end + +function decode_plain_fixed(bytes::AbstractVector{UInt8}, count::Integer, width::Integer; + offset::Integer=1, limits::Limits=Limits()) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + width >= 0 || throw(ArgumentError("fixed byte-array width must be nonnegative")) + _checklimit(:container_elements, count, limits.max_container_elements) + _checklimit(:string_bytes, width, limits.max_string_bytes) + total = Base.checked_mul(Int(count), Int(width)) + position = Int(offset) + _requirebytes(bytes, position, total) + output = Matrix{UInt8}(undef, Int(width), Int(count)) + isempty(output) || copyto!(output, 1, bytes, position, total) + return output, position + total +end + +function encode_plain(values::AbstractVector{Bool}) + output = zeros(UInt8, cld(length(values), 8)) + @inbounds for index in eachindex(values) + values[index] || continue + zeroindex = index - 1 + output[(zeroindex >> 3) + 1] |= UInt8(1) << (zeroindex & 7) + end + return output +end + +function encode_plain(values::AbstractVector{T}) where {T<:Union{Int32,Int64,Float32,Float64}} + output = UInt8[] + sizehint!(output, Base.checked_mul(length(values), sizeof(T))) + U = _plainbits(T) + for value in values + _writelittle!(output, _toplainbits(U, value)) + end + return output +end + +function encode_plain_byte_array(values) + output = UInt8[] + for value in values + bytes = value isa AbstractString ? codeunits(value) : value + length(bytes) <= typemax(Int32) || throw(ArgumentError("byte array exceeds Int32 length")) + _writelittle!(output, reinterpret(UInt32, Int32(length(bytes)))) + append!(output, bytes) + end + return output +end + +function encode_plain_fixed(values::AbstractMatrix{UInt8}) + return collect(vec(values)) +end diff --git a/src/reader.jl b/src/reader.jl deleted file mode 100644 index f1bba1f..0000000 --- a/src/reader.jl +++ /dev/null @@ -1,464 +0,0 @@ - -const PAR_MAGIC = "PAR1" -const SZ_PAR_MAGIC = length(PAR_MAGIC) -const SZ_FOOTER = 4 -const SZ_VALID_PAR = 2*SZ_PAR_MAGIC + SZ_FOOTER - -# page is the unit of compression -mutable struct Page - colchunk::ColumnChunk - hdr::PageHeader - pos::Int - uncompressed_data::Vector{UInt8} - nextpos::Int64 -end - -""" -Keeps a cache of pages read from a file. -Pages are kept as weak refs, so that they can be collected when there's memory pressure. -""" -struct PageLRU - refs::Dict{Tuple{ColumnChunk,Int64},WeakRef} - lck::ReentrantLock - function PageLRU() - new(Dict{Tuple{ColumnChunk,Int64},WeakRef}(), ReentrantLock()) - end -end - -function cacheget(fetcher, lru::PageLRU, chunk::ColumnChunk, startpos::Int64) - key = (chunk,startpos) - lock(lru.lck) do - filter!(kv->(kv[2].value !== nothing), lru.refs) - page = haskey(lru.refs, key) ? lru.refs[key].value : nothing - if page === nothing - page = fetcher()::Page - lru.refs[key] = WeakRef(page) - end - return page - end -end - -""" - Parquet.File(path; map_logical_types) => Parquet.File - -Represents a Parquet file at `path` open for reading. Options to map logical types can be provided via `map_logical_types`. - -`map_logical_types` can be one of: - -- `false`: no mapping is done (default) -- `true`: default mappings are attempted on all columns (bytearray => String, int96 => DateTime) -- A user supplied dict mapping column names to a tuple of type and a converter function - -Returns a `Parquet.File` type that keeps a handle to the open file and the file metadata and also holds a LRU cache of raw bytes of the pages read. -""" -mutable struct File - path::String - handle::IOStream - meta::FileMetaData - schema::Schema - page_cache::PageLRU -end - -function File(path::AbstractString; map_logical_types::Dict=TLogicalTypeMap()) - f = open(path) - try - return File(path, f; map_logical_types=map_logical_types) - catch ex - close(f) - rethrow() - end -end - -function File(path::AbstractString, handle::IOStream; map_logical_types::Dict=TLogicalTypeMap()) - is_par_file(handle) || error("Not a parquet format file: $path") - meta_len = metadata_length(handle) - meta = metadata(handle, path, meta_len) - typemap = merge!(TLogicalTypeMap(), map_logical_types) - file = File(String(path), handle, meta, Schema(meta.schema, typemap), PageLRU()) - finalizer(close, file) - file -end - -function close(par::Parquet.File) - empty!(par.page_cache.refs) - close(par.handle) -end - -schema(par::Parquet.File) = par.schema - -colname(par::Parquet.File, col::ColumnChunk) = colname(metadata(par,col)) -colname(col::ColumnMetaData) = col.path_in_schema -function colnames(par::Parquet.File) - names = Vector{Vector{String}}() - cs = Int[] - ns = String[] - for x in par.schema.schema[2:end] - if Parquet.num_children(x) > 0 - push!(cs, x.num_children) - push!(ns, x.name) - else - if !isempty(cs) - push!(names, [ns; x.name]) - cs[end] -= 1 - if cs[end] == 0 - pop!(cs) - pop!(ns) - end - else - push!(names, [x.name]) - end - end - end - names -end - -ncols(par::Parquet.File) = length(colnames(par)) -nrows(par::Parquet.File) = par.meta.num_rows - -coltype(par::Parquet.File, col::ColumnChunk) = coltype(metadata(par,col)) -coltype(col::ColumnMetaData) = col._type - -# return all rowgroups in the par file -rowgroups(par::Parquet.File) = par.meta.row_groups - -function rowgroup_row_positions(par::Parquet.File) - rgs = rowgroups(par) - positions = Array{Int64}(undef, length(rgs)+1) - idx = 1 - positions[idx] = Int64(1) - for rg in rgs - positions[idx+=1] = rg.num_rows - end - cumsum!(positions, positions) -end - -columns(par::Parquet.File, rowgroupidx) = columns(par, rowgroups(par)[rowgroupidx]) -columns(par::Parquet.File, rowgroup::RowGroup) = rowgroup.columns -columns(par::Parquet.File, rowgroup::RowGroup, colname::Vector{String}) = columns(par, rowgroup, [colname]) -function columns(par::Parquet.File, rowgroup::RowGroup, cnames::Vector{Vector{String}}) - R = ColumnChunk[] - for col in columns(par, rowgroup) - (colname(par,col) in cnames) && push!(R, col) - end - R -end - -## -# Iterator for pages in a column chunk -mutable struct ColumnChunkPages - par::Parquet.File - col::ColumnChunk - startpos::Int64 - endpos::Int64 - - function ColumnChunkPages(par::Parquet.File, col::ColumnChunk) - startpos = page_offset(par, col) - endpos = end_offset(par, col) - new(par, col, startpos, endpos) - end -end -eltype(::Type{ColumnChunkPages}) = Page -Base.iterate(ccp::ColumnChunkPages) = iterate(ccp, ccp.startpos) -function Base.iterate(ccp::ColumnChunkPages, startpos::Int64) - if startpos >= ccp.endpos - return nothing - end - - page = cacheget(ccp.par.page_cache, ccp.col, startpos) do - par = ccp.par - io = par.handle - seek(io, startpos) - pagehdr = read_thrift(io, PageHeader) - - page_data_pos = position(io) - pagesz = page_size(pagehdr) - data = _use_mmap[] ? Mmap.mmap(io, Vector{UInt8}, (pagesz,), page_data_pos; grow=false, shared=false) : read!(io, Array{UInt8}(undef, pagesz)) - codec = metadata(ccp.par, ccp.col).codec - - if (codec != CompressionCodec.UNCOMPRESSED) - uncompressed_sz = pagehdr.uncompressed_page_size - #uncompressed_data = _use_mmap[] ? Mmap.mmap(Mmap.Anonymous(), Vector{UInt8}, (uncompressed_sz,), 0) : Array{UInt8}(undef, uncompressed_sz) - uncompressed_data = Array{UInt8}(undef, uncompressed_sz) - if codec == CompressionCodec.SNAPPY - Snappy.snappy_uncompress(data, uncompressed_data) - elseif codec == CompressionCodec.GZIP - readbytes!(GzipDecompressorStream(IOBuffer(data)), uncompressed_data) - elseif codec == CompressionCodec.ZSTD - readbytes!(ZstdDecompressorStream(IOBuffer(data)), uncompressed_data) - else - error("Unknown compression codec for column chunk: $codec") - end - (length(uncompressed_data) == uncompressed_sz) || error("failed to uncompress page. expected $(uncompressed_sz), got $(length(uncompressed_data)) bytes") - else - uncompressed_data = data - end - - nextpos = page_data_pos + pagesz - page = Page(ccp.col, pagehdr, page_data_pos, uncompressed_data, nextpos) - end - - page, page.nextpos -end - -## -# Iterator for page values in a column chunk -mutable struct ColumnChunkPageValues{T} - ccp::ColumnChunkPages - max_repn::Int64 - max_defn::Int64 - has_repn_levels::Bool - has_defn_levels::Bool - repn_out::OutputState{Int32} - defn_out::OutputState{Int32} - valdict_out::OutputState{T} - vals_out::OutputState{T} - converter_fn::Function -end - -function ColumnChunkPageValues(par::Parquet.File, col::ColumnChunk, ::Type{T}, converter_fn::Function=identity) where {T} - cname = colname(par, col) - - max_repn = max_repetition_level(par.schema, cname) - max_defn = max_definition_level(par.schema, cname) - has_repn_levels = ((length(cname) > 1) && (max_repn > 0)) - has_defn_levels = !isrequired(par.schema, cname) - - repn_out = OutputState(Int32, 0) - defn_out = OutputState(Int32, 0) - valdict_out = OutputState(T, 0) - vals_out = OutputState(T, 0) - - ccp = ColumnChunkPages(par, col) - ColumnChunkPageValues{T}(ccp, max_repn, max_defn, has_repn_levels, has_defn_levels, repn_out, defn_out, valdict_out, vals_out, converter_fn) -end - -function eltype(::Type{ColumnChunkPageValues{T}}) where {T} - NamedTuple{(:value,:repn_level,:defn_level),Tuple{OutputState{T},OutputState{Int32},OutputState{Int32}}} -end - -function Base.iterate(ccpv::ColumnChunkPageValues{T}) where {T} - iterate(ccpv, ccpv.ccp.startpos) -end - -function map_dict_vals(valdict::OutputState{T1}, vals::OutputState{T1}, map_vals::Vector{T2}) where {T1, T2} - if !isempty(valdict.data) && (valdict.offset > 0) - num_values = length(map_vals) - @inbounds for idx in 1:num_values - vals.data[idx] = valdict.data[map_vals[idx]+1] - end - vals.offset += num_values - end -end - -function Base.iterate(ccpv::ColumnChunkPageValues{T}, startpos::Int64) where {T} - if startpos >= ccpv.ccp.endpos - return nothing - end - - read_data_page = false - nextpos = startpos - - while !read_data_page - page, nextpos = iterate(ccpv.ccp, nextpos) - - pagetype = page.hdr._type - num_values = page_num_values(page) - inp = InputState(page.uncompressed_data, 0) - - if (pagetype === PageType.DATA_PAGE) || (pagetype === PageType.DATA_PAGE_V2) - read_data_page = true - ccpv.has_repn_levels && reset_to_size(ccpv.repn_out, num_values) - ccpv.has_defn_levels && reset_to_size(ccpv.defn_out, num_values) - - @debug("reading a data page for columnchunk") - enc, defn_enc, repn_enc = page_encodings(page) - nmissing = read_levels_and_nmissing(inp, ccpv.defn_out, ccpv.repn_out, defn_enc, repn_enc, Int(ccpv.max_defn), Int(ccpv.max_repn), num_values) - nnonmissing = num_values - nmissing - reset_to_size(ccpv.vals_out, nnonmissing) - - if enc === Encoding.PLAIN_DICTIONARY || enc === Encoding.RLE_DICTIONARY - map_vals = read_data_dict(inp, nnonmissing) - map_dict_vals(ccpv.valdict_out, ccpv.vals_out, map_vals) - else - if ccpv.converter_fn === identity - read_plain_values(inp, ccpv.vals_out, nnonmissing) - else - read_plain_values(inp, ccpv.vals_out, nnonmissing, ccpv.converter_fn, ccpv.ccp.col.meta_data._type) - end - end - elseif pagetype === PageType.DICTIONARY_PAGE - ensure_additional_size(ccpv.valdict_out, num_values) - if ccpv.converter_fn === identity - read_plain_values(inp, ccpv.valdict_out, num_values) - else - read_plain_values(inp, ccpv.valdict_out, num_values, ccpv.converter_fn, ccpv.ccp.col.meta_data._type) - end - else - error("unsupported page type $typ") - end - end - iterator_result = NamedTuple{(:value,:repn_level,:defn_level),Tuple{OutputState{T},OutputState{Int32},OutputState{Int32}}}((ccpv.vals_out, ccpv.repn_out, ccpv.defn_out)) - iterator_result, nextpos -end - -## -# layer 2 access -# can access decoded values from pages -function read_levels(inp::InputState, out::OutputState{Int32}, max_val::Int, enc::Int32, num_values::Int32) # levels are always 32 bits - bit_width = UInt8(@bitwidth(max_val)) - @assert(bit_width !== 0) - #@debug("reading levels. enc:$enc ($(Thrift.enumstr(Encoding,enc))), max_val:$max_val, num_values:$num_values") - - if enc === Encoding.RLE - byte_width = @bit2bytewidth(bit_width) - read_hybrid(inp, out, num_values, bit_width, byte_width) - elseif enc === Encoding.BIT_PACKED - read_bitpacked_run_old(inp, out, num_values, bit_width) - else - error("unsupported encoding $enc ($(Thrift.enumstr(Encoding,enc))) for levels") - end -end - -function read_levels_and_nmissing(inp::InputState, defn_out::OutputState{Int32}, repn_out::OutputState{Int32}, defn_enc::Int32, repn_enc::Int32, max_defn::Int, max_repn::Int, num_values::Int32) - # read repetition levels. skipped if all columns are at 1st level - if !isempty(repn_out.data) - read_levels(inp, repn_out, max_repn, repn_enc, num_values) - end - - # read definition levels. skipped if column is required - nmissing = Int32(0) - if !isempty(defn_out.data) - defn_levels = defn_out.data - defn_offset = defn_out.offset - read_levels(inp, defn_out, max_defn, defn_enc, num_values) - @inbounds for idx in 1:num_values - (defn_levels[idx+defn_offset] === Int32(0)) && (nmissing += Int32(1)) - end - end - - nmissing -end - - -# column and page metadata -open(par::Parquet.File, col::ColumnChunk) = open(par.handle, par.path, col) -close(par::Parquet.File, col::ColumnChunk, io) = (par.handle == io) || close(io) -function open(io, path::AbstractString, col::ColumnChunk) - if hasproperty(col, :file_path) - @debug("opening file to read column metadata", file=col.file_path, offset=col.file_offset) - open(col.file_path) - else - if io === nothing - @debug("opening file to read column metadata", file=path, offset=col.file_offset) - open(path) - else - @debug("reading column metadata", offset=col.file_offset) - io - end - end -end - -function metadata(io, path::AbstractString, col::ColumnChunk) - fio = open(io, path, col) - seek(fio, col.file_offset) - meta = read_thrift(fio, ColumnMetaData) - (fio !== io) && close(fio) - meta -end - -function page_offset(par::Parquet.File, col::ColumnChunk) - colmeta = metadata(par, col) - offset = colmeta.data_page_offset - hasproperty(colmeta, :index_page_offset) && (offset = min(offset, colmeta.index_page_offset)) - hasproperty(colmeta, :dictionary_page_offset) && (offset = min(offset, colmeta.dictionary_page_offset)) - offset -end -end_offset(par::Parquet.File, col::ColumnChunk) = page_offset(par, col) + metadata(par,col).total_compressed_size - -page_size(page::PageHeader) = hasproperty(page, :compressed_page_size) ? page.compressed_page_size : page.uncompressed_page_size - -const INVALID_ENC = Int32(-1) -page_encodings(page::Page) = page_encodings(page.hdr) -function page_encodings(page::PageHeader) - hasproperty(page, :data_page_header) ? page_encodings(page.data_page_header) : - hasproperty(page, :data_page_header_v2) ? page_encodings(page.data_page_header_v2) : - hasproperty(page, :dictionary_page_header) ? page_encodings(page.dictionary_page_header) : - (INVALID_ENC,INVALID_ENC,INVALID_ENC) -end -page_encodings(page::DictionaryPageHeader) = (page.encoding,INVALID_ENC,INVALID_ENC) -page_encodings(page::DataPageHeader) = (page.encoding, page.definition_level_encoding, page.repetition_level_encoding) -page_encodings(page::DataPageHeaderV2) = (page.encoding, Encoding.RLE, Encoding.RLE) - -page_num_values(page::Page) = page_num_values(page.hdr) -function page_num_values(page::PageHeader) - hasproperty(page, :data_page_header) ? page_num_values(page.data_page_header) : - hasproperty(page, :data_page_header_v2) ? page_num_values(page.data_page_header_v2) : - hasproperty(page, :dictionary_page_header) ? page_num_values(page.dictionary_page_header) : Int32(0) -end -page_num_values(page::Union{DataPageHeader,DataPageHeaderV2,DictionaryPageHeader}) = page.num_values - -# file metadata -read_thrift(buff::Array{UInt8}, ::Type{T}) where {T} = read(TCompactProtocol(TMemoryTransport(buff)), T) -read_thrift(io::IO, ::Type{T}) where {T} = read(TCompactProtocol(TFileTransport(io)), T) -read_thrift(t::TR, ::Type{T}) where {TR<:TTransport,T} = read(TCompactProtocol(t), T) - -function metadata_length(io) - sz = filesize(io) - seek(io, sz - SZ_PAR_MAGIC - SZ_FOOTER) - - # read footer size as little endian signed Int32 - read_fixed(io, Int32) -end - -function metadata(io, path::AbstractString, len::Integer=metadata_length(io)) - @debug("reading file metadata", len) - sz = filesize(io) - seek(io, sz - SZ_PAR_MAGIC - SZ_FOOTER - len) - meta = read_thrift(io, FileMetaData) - meta -end - -metadata(par::Parquet.File) = par.meta - -#= -function fill_column_metadata(par::Parquet.File) - meta = par.meta - # go through all column chunks and read metadata from file offsets if required - for grp in meta.row_groups - for col in grp.columns - metadata(par, col) - end - end -end -=# - -function metadata(par::Parquet.File, col::ColumnChunk) - if !hasproperty(col, :meta_data) - col.meta_data = metadata(par.handle, par.path, col) - end - col.meta_data -end - -# file format verification -function is_par_file(fname::AbstractString) - open(fname) do io - return is_par_file(io) - end -end - -function is_par_file(io) - sz = filesize(io) - (sz > SZ_VALID_PAR) || return false - - seekstart(io) - magic = Array{UInt8}(undef, 4) - read!(io, magic) - (String(magic) == PAR_MAGIC) || return false - - seek(io, sz - SZ_PAR_MAGIC) - magic = Array{UInt8}(undef, 4) - read!(io, magic) - (String(magic) == PAR_MAGIC) || return false - - true -end diff --git a/src/rle.jl b/src/rle.jl new file mode 100644 index 0000000..3231343 --- /dev/null +++ b/src/rle.jl @@ -0,0 +1,184 @@ +function _readhybridvarint(bytes::AbstractVector{UInt8}, offset::Int) + value = UInt64(0) + position = offset + for index in 0:9 + _requirebytes(bytes, position, 1) + byte = bytes[position] + position += 1 + index == 9 && byte > 0x01 && throw(FormatError("hybrid run header overflows UInt64")) + value |= UInt64(byte & 0x7f) << (7 * index) + iszero(byte & 0x80) && return value, position + end + throw(FormatError("unterminated hybrid run header")) +end + +function _writehybridvarint!(output::Vector{UInt8}, value::UInt64) + while value >= 0x80 + push!(output, UInt8(value & 0x7f) | 0x80) + value >>= 7 + end + push!(output, UInt8(value)) + return +end + +function _readpackedvalue(bytes::AbstractVector{UInt8}, offset::Int, bitoffset::Int, + bitwidth::Int) + value = UInt64(0) + bitwidth == 0 && return value + @inbounds for bit in 0:(bitwidth - 1) + absolute = bitoffset + bit + byte = bytes[offset + (absolute >> 3)] + value |= UInt64((byte >> (absolute & 7)) & 0x01) << bit + end + return value +end + +function _decodebitpacked!(output::Vector{UInt64}, outputoffset::Int, + bytes::AbstractVector{UInt8}, offset::Int, groups::Int, bitwidth::Int) + payloadbytes = try + Base.checked_mul(groups, bitwidth) + catch err + err isa OverflowError || rethrow() + throw(FormatError("bit-packed payload size overflows Int")) + end + _requirebytes(bytes, offset, payloadbytes) + available = try + Base.checked_mul(groups, 8) + catch err + err isa OverflowError || rethrow() + throw(FormatError("bit-packed value count overflows Int")) + end + count = min(available, length(output) - outputoffset + 1) + @inbounds for index in 0:(count - 1) + output[outputoffset + index] = _readpackedvalue(bytes, offset, index * bitwidth, bitwidth) + end + return count, offset + payloadbytes +end + +function _decoderlerun!(output::Vector{UInt64}, outputoffset::Int, + bytes::AbstractVector{UInt8}, offset::Int, runlength::Int, bitwidth::Int) + width = cld(bitwidth, 8) + _requirebytes(bytes, offset, width) + value = UInt64(0) + @inbounds for index in 0:(width - 1) + value |= UInt64(bytes[offset + index]) << (8 * index) + end + count = min(runlength, length(output) - outputoffset + 1) + fill!(@view(output[outputoffset:(outputoffset + count - 1)]), value) + return count, offset + width +end + +function decode_hybrid(bytes::AbstractVector{UInt8}, count::Integer, bitwidth::Integer; + offset::Integer=1, length_prefix::Bool=false, limits::Limits=Limits()) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + 0 <= bitwidth <= 64 || throw(ArgumentError("bit width must be between 0 and 64")) + _checklimit(:container_elements, count, limits.max_container_elements) + position = Int(offset) + endposition = length(bytes) + 1 + if length_prefix + rawlength, position = _readlittle(UInt32, bytes, position) + bodylength = Int(rawlength) + _requirebytes(bytes, position, bodylength) + endposition = position + bodylength + end + output = Vector{UInt64}(undef, Int(count)) + outputoffset = 1 + while outputoffset <= length(output) + position < endposition || throw(FormatError("hybrid stream ended before all values")) + header, position = _readhybridvarint(bytes, position) + rawlength = header >> 1 + rawlength > 0 || throw(FormatError("zero-length hybrid run")) + if isodd(header) + rawlength <= UInt64(typemax(Int32) ÷ 8) || + throw(FormatError("bit-packed hybrid run exceeds Int32 values")) + runlength = Int(rawlength) + written, position = _decodebitpacked!(output, outputoffset, bytes, position, + runlength, Int(bitwidth)) + else + rawlength <= UInt64(typemax(Int32)) || + throw(FormatError("RLE hybrid run exceeds Int32 values")) + runlength = Int(rawlength) + written, position = _decoderlerun!(output, outputoffset, bytes, position, + runlength, Int(bitwidth)) + end + position <= endposition || throw(FormatError("hybrid run exceeds its declared length")) + outputoffset += written + end + length_prefix && position != endposition && throw(FormatError("hybrid stream has trailing bytes")) + return output, position +end + +function _setpackedvalue!(output::AbstractVector{UInt8}, bitoffset::Int, bitwidth::Int, value::UInt64) + bitwidth == 0 && return + @inbounds for bit in 0:(bitwidth - 1) + iszero(value & (UInt64(1) << bit)) && continue + absolute = bitoffset + bit + output[(absolute >> 3) + 1] |= UInt8(1) << (absolute & 7) + end + return +end + +function _hybridbody(values, bitwidth::Int) + groups = cld(length(values), 8) + groups == 0 && return UInt8[] + output = UInt8[] + _writehybridvarint!(output, (UInt64(groups) << 1) | 0x01) + payloadstart = length(output) + append!(output, zeros(UInt8, Base.checked_mul(groups, bitwidth))) + limit = bitwidth == 64 ? typemax(UInt64) : (UInt64(1) << bitwidth) - 1 + for (index, rawvalue) in enumerate(values) + rawvalue >= 0 || throw(ArgumentError("hybrid values must be nonnegative")) + value = UInt64(rawvalue) + value <= limit || throw(ArgumentError("hybrid value does not fit the bit width")) + target = @view output[(payloadstart + 1):end] + _setpackedvalue!(target, (index - 1) * bitwidth, bitwidth, value) + end + return output +end + +function encode_hybrid(values, bitwidth::Integer; length_prefix::Bool=false) + 0 <= bitwidth <= 64 || throw(ArgumentError("bit width must be between 0 and 64")) + body = _hybridbody(values, Int(bitwidth)) + length_prefix || return body + length(body) <= typemax(UInt32) || throw(ArgumentError("hybrid stream exceeds UInt32 length")) + output = UInt8[] + _writelittle!(output, UInt32(length(body))) + append!(output, body) + return output +end + +function decode_bit_packed(bytes::AbstractVector{UInt8}, count::Integer, bitwidth::Integer; + offset::Integer=1, limits::Limits=Limits()) + count >= 0 || throw(ArgumentError("value count must be nonnegative")) + 0 <= bitwidth <= 64 || throw(ArgumentError("bit width must be between 0 and 64")) + count <= typemax(Int) || throw(FormatError("bit-packed value count overflows Int")) + _checklimit(:container_elements, count, limits.max_container_elements) + size = Int(count) + width = Int(bitwidth) + totalbits = try + Base.checked_mul(size, width) + catch err + err isa OverflowError || rethrow() + throw(FormatError("bit-packed payload size overflows Int")) + end + bytecount = cld(totalbits, 8) + position = Int(offset) + _requirebytes(bytes, position, bytecount) + output = Vector{UInt64}(undef, size) + @inbounds for index in 0:(size - 1) + value = UInt64(0) + for bit in 0:(width - 1) + absolute = index * width + bit + packed = (bytes[position + (absolute >> 3)] >> (7 - (absolute & 7))) & 0x01 + value = (value << 1) | UInt64(packed) + end + output[index + 1] = value + end + padding = bytecount * 8 - totalbits + if padding > 0 + mask = UInt8((UInt16(1) << padding) - 1) + iszero(bytes[position + bytecount - 1] & mask) || + throw(FormatError("BIT_PACKED stream has nonzero padding bits")) + end + return output, position + bytecount +end diff --git a/src/schema.jl b/src/schema.jl index 5d1ea45..8b6ec71 100644 --- a/src/schema.jl +++ b/src/schema.jl @@ -1,223 +1,188 @@ -# A logical type map can be provided during schema construction. -# It contains mapping of a column to a logical type and the converter function to be applied. -# Columns can be indentified either by their actual type or column name (the full path in the schema) -const TLogicalTypeMap = Dict{Union{Int32,Vector{String}},Tuple{DataType,Function}} - -# schema and helper methods -mutable struct Schema - schema::Vector{SchemaElement} - map_logical_types::TLogicalTypeMap - name_lookup::Dict{Vector{String},SchemaElement} - type_lookup::Dict{Vector{String},Union{DataType,Union}} - nttype_lookup::Dict{Vector{String},Union{DataType,Union}} - - function Schema(elems::Vector{SchemaElement}, map_logical_types::TLogicalTypeMap=TLogicalTypeMap()) - name_lookup = Dict{Vector{String},SchemaElement}() - name_stack = String[] - nchildren_stack = Int[] - - for idx in 1:length(elems) - sch = elems[idx] - nested_name = [name_stack; sch.name] - name_lookup[nested_name] = sch - - if !haskey(map_logical_types, nested_name) - if is_logical_string(sch) - map_logical_types[nested_name] = (String, logical_string) - elseif is_logical_timestamp(sch) - map_logical_types[nested_name] = (DateTime, logical_timestamp) - elseif is_logical_decimal(sch) - map_logical_types[nested_name] = map_logical_decimal(sch.precision, sch.scale) - end - end - - if (idx > 1) && (num_children(sch) > 0) - push!(nchildren_stack, sch.num_children) - push!(name_stack, sch.name) - elseif !isempty(nchildren_stack) - if nchildren_stack[end] == 1 - pop!(nchildren_stack) - pop!(name_stack) - else - nchildren_stack[end] -= 1 - end - end - end - new(elems, map_logical_types, name_lookup, Dict{Vector{String},Union{DataType,Union}}(), Dict{Vector{String},Union{DataType,Union}}()) - end +struct SchemaNode + element::Metadata.SchemaElement + path::Vector{String} + max_definition_level::Int16 + max_repetition_level::Int16 + column_index::Int32 + children::Vector{SchemaNode} end -leafname(schname::T) where {T <: AbstractVector{String}} = [schname[end]] - -parentname(schname::T) where {T <: AbstractVector{String}} = istoplevel(schname) ? schname : schname[1:(end-1)] - -istoplevel(schname::Vector) = !(length(schname) > 1) - -elem(sch::Schema, schname::T) where {T <: AbstractVector{String}} = sch.name_lookup[schname] -function elemindex(sch::Schema, schname::T) where {T <: AbstractVector{String}} - schema_element = elem(sch, schname) - findfirst(x->x===schema_element, sch.schema) +struct Schema + root::SchemaNode + leaves::Vector{SchemaNode} end -isrepetitiontype(schelem::SchemaElement, repetition_type) = hasproperty(schelem, :repetition_type) && (schelem.repetition_type == repetition_type) - -isrequired(sch::Schema, schname::T) where {T <: AbstractVector{String}} = isrequired(elem(sch, schname)) -isrequired(schelem::SchemaElement) = isrepetitiontype(schelem, FieldRepetitionType.REQUIRED) - -isoptional(sch::Schema, schname::T) where {T <: AbstractVector{String}} = isoptional(elem(sch, schname)) -isoptional(schelem::SchemaElement) = isrepetitiontype(schelem, FieldRepetitionType.OPTIONAL) - -isrepeated(sch::Schema, schname::T) where {T <: AbstractVector{String}} = isrepeated(elem(sch, schname)) -isrepeated(schelem::SchemaElement) = isrepetitiontype(schelem, FieldRepetitionType.REPEATED) - -is_logical_string(sch::SchemaElement) = hasproperty(sch, :_type) && (sch._type === _Type.BYTE_ARRAY) && ((hasproperty(sch, :converted_type) && (sch.converted_type === ConvertedType.UTF8)) || (hasproperty(sch, :logicalType) && hasproperty(sch.logicalType, :STRING))) - -# converted_type is usually not set for INT96 types, but they are used exclusively used for timestamps only -is_logical_timestamp(sch::SchemaElement) = hasproperty(sch, :_type) && (sch._type === _Type.INT96) - -function is_logical_decimal(sch::SchemaElement) - if hasproperty(sch, :_type) - if (sch._type === _Type.FIXED_LEN_BYTE_ARRAY) || (sch._type === _Type.INT64) - if (hasproperty(sch, :converted_type) && (sch.converted_type === ConvertedType.DECIMAL)) || (hasproperty(sch, :logicalType) && hasproperty(sch.logicalType, :DECIMAL)) - return true - end - end +function _schemalevels(element::Metadata.SchemaElement, definition::Integer, + repetition::Integer) + kind = element.repetition_type + kind === nothing && throw(FormatError("schema element $(repr(element.name)) has no repetition type")) + if kind == Metadata.FieldRepetitionType.REQUIRED + return (definition, repetition) + elseif kind == Metadata.FieldRepetitionType.OPTIONAL + return (definition + 1, repetition) + elseif kind == Metadata.FieldRepetitionType.REPEATED + return (definition + 1, repetition + 1) end - false + throw(FormatError("schema element $(repr(element.name)) has an unknown repetition type $(kind.value)")) end -function path_in_schema(sch::Schema, schelem::SchemaElement) - for (n,v) in sch.name_lookup - (v === schelem) && return n - end - error("schema element not found in schema") +function _schemachildcount(element::Metadata.SchemaElement) + count = element.num_children + count === nothing && throw(FormatError("group schema element $(repr(element.name)) has no child count")) + count >= 0 || throw(FormatError("group schema element $(repr(element.name)) has a negative child count")) + return Int(count) end -function logical_converter(sch::Schema, schname::T) where {T <: AbstractVector{String}} - elem = sch.name_lookup[schname] - - if schname in keys(sch.map_logical_types) - _logical_type, converter = sch.map_logical_types[schname] - return converter - elseif hasproperty(elem, :_type) && (elem._type in keys(sch.map_logical_types)) - _logical_type, converter = sch.map_logical_types[elem._type] - return converter - else - return identity +function _validateschemashape(element::Metadata.SchemaElement, limits::Limits) + if element.type_ === nothing + return _schemachildcount(element) end -end - -function logical_convert(sch::Schema, schname::T, val) where {T <: AbstractVector{String}} - elem = sch.name_lookup[schname] - - if schname in keys(sch.map_logical_types) - logical_type, converter = sch.map_logical_types[schname] - converter(val)::logical_type - elseif hasproperty(elem, :_type) && (elem._type in keys(sch.map_logical_types)) - logical_type, converter = sch.map_logical_types[elem._type] - converter(val)::logical_type - else - val + children = something(element.num_children, Int32(0)) + children == 0 || throw(FormatError("primitive schema element $(repr(element.name)) has children")) + if element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY + length = element.type_length + length !== nothing && length > 0 || + throw(FormatError("fixed-length schema element $(repr(element.name)) has no positive length")) + _checklimit(:string_bytes, length, limits.max_string_bytes) end + return 0 end -elemtype(sch::Schema, schname::T) where {T <: AbstractVector{String}} = get!(sch.type_lookup, schname) do - elem = sch.name_lookup[schname] +mutable struct _SchemaParseFrame + element::Metadata.SchemaElement + path::Vector{String} + definition::Int16 + repetition::Int16 + children::Vector{SchemaNode} + expected::Int + completed::Int + parent::Union{Nothing,_SchemaParseFrame} +end - if schname in keys(sch.map_logical_types) - logical_type, _converter = sch.map_logical_types[schname] - logical_type - elseif hasproperty(elem, :_type) && (elem._type in keys(sch.map_logical_types)) - logical_type, _converter = sch.map_logical_types[elem._type] - logical_type - else - elemtype(elem) - end +function _schemapath(parent::Vector{String}, name::String, + budget::_LiveByteBudget) + count = length(parent) + 1 + _reservearray!(budget, String, count) + path = Vector{String}(undef, count) + copyto!(path, 1, parent, 1, length(parent)) + path[end] = name + return path end -function elemtype(schelem::SchemaElement) - jtype = Nothing - if hasproperty(schelem, :_type) - jtype = PLAIN_JTYPES[schelem._type+1] - else - jtype = Dict{Symbol,Any} # this is a nested type +function _schemaparsestart(elements::Vector{Metadata.SchemaElement}, index::Int, + parentpath::Vector{String}, definition::Integer, repetition::Integer, + depth::Int, + leaves::Vector{SchemaNode}, limits::Limits, budget::_LiveByteBudget, + parent; root::Bool=false) + index <= length(elements) || throw(FormatError("flattened schema ends before all declared children")) + element = elements[index] + children = _validateschemashape(element, limits) + nextdefinition, nextrepetition = root ? (definition, repetition) : + _schemalevels(element, definition, repetition) + if element.type_ === nothing + children <= length(elements) - index || throw(FormatError( + "group schema element $(repr(element.name)) declares more direct " * + "children than remain in the flattened schema")) end - - if (hasproperty(schelem, :_type) && (schelem._type == _Type.BYTE_ARRAY || schelem._type == _Type.FIXED_LEN_BYTE_ARRAY)) || - (hasproperty(schelem, :repetition_type) && (schelem.repetition_type == FieldRepetitionType.REPEATED)) # array type - jtype = Vector{jtype} + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + nextdefinition <= typemax(Int16) || throw(FormatError("schema definition level exceeds Int16")) + nextrepetition <= typemax(Int16) || throw(FormatError("schema repetition level exceeds Int16")) + path = root ? parentpath : _schemapath(parentpath, element.name, budget) + if element.type_ !== nothing + ordinal = try + Base.checked_add(length(leaves), 1) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + Int64(typemax(Int32)))) + end + ordinal <= typemax(Int32) || throw(LimitError(:container_elements, + ordinal, Int64(typemax(Int32)))) + column = Int32(ordinal) + _reservearray!(budget, SchemaNode, 0) + _reserveobjects!(budget) + node = SchemaNode(element, path, Int16(nextdefinition), Int16(nextrepetition), column, SchemaNode[]) + push!(leaves, node) + return (node, nothing, index + 1) end - - jtype + _checklimit(:container_elements, children, + limits.max_container_elements) + _reservearray!(budget, SchemaNode, children) + nodes = SchemaNode[] + sizehint!(nodes, children) + _reserveobjects!(budget) + frame = _SchemaParseFrame(element, path, Int16(nextdefinition), + Int16(nextrepetition), nodes, children, 0, parent) + return (nothing, frame, index + 1) end -ntcolstype(sch::Schema, schname::T) where {T <: AbstractVector{String}} = get!(sch.nttype_lookup, schname) do - ntcolstype(sch, sch.name_lookup[schname]) -end -function ntcolstype(sch::Schema, schelem::SchemaElement) - @assert num_children(schelem) > 0 - idx = findfirst(x->x===schelem, sch.schema) - children_range = (idx+1):(idx+schelem.num_children) - names = [Symbol(x.name) for x in sch.schema[children_range]] - types = [(num_children(x) > 0) ? ntelemtype(sch, path_in_schema(sch, x)) : elemtype(sch, path_in_schema(sch, x)) for x in sch.schema[children_range]] - optionals = [isoptional(x) for x in sch.schema[children_range]] - types = [Vector{opt ? Union{t,Missing} : t} for (t,opt) in zip(types, optionals)] - NamedTuple{(names...,),Tuple{types...}} -end - -ntelemtype(sch::Schema, schname::T) where {T <: AbstractVector{String}} = get!(sch.nttype_lookup, schname) do - ntelemtype(sch, sch.name_lookup[schname]) -end -function ntelemtype(sch::Schema, schelem::SchemaElement) - @assert num_children(schelem) > 0 - idx = findfirst(x->x===schelem, sch.schema) - children_range = (idx+1):(idx+schelem.num_children) - repeated = hasproperty(schelem, :repetition_type) && (schelem.repetition_type == FieldRepetitionType.REPEATED) - names = [Symbol(x.name) for x in sch.schema[children_range]] - types = [(num_children(x) > 0) ? ntelemtype(sch, path_in_schema(sch, x)) : elemtype(sch, path_in_schema(sch, x)) for x in sch.schema[children_range]] - optionals = [isoptional(x) for x in sch.schema[children_range]] - types = [opt ? Union{t,Missing} : t for (t,opt) in zip(types, optionals)] - T = NamedTuple{(names...,),Tuple{types...}} - repeated ? Vector{T} : T +function _parseschema(elements::Vector{Metadata.SchemaElement}, + leaves::Vector{SchemaNode}, rootpath::Vector{String}, limits::Limits, + budget::_LiveByteBudget) + node, frame, nextindex = _schemaparsestart(elements, 1, rootpath, 0, 0, 1, + leaves, limits, budget, nothing; root=true) + node === nothing || throw(AssertionError("schema root parser returned a primitive")) + current = frame::_SchemaParseFrame + while true + if current.completed == current.expected + _reserveobjects!(budget) + completed = SchemaNode(current.element, current.path, + current.definition, current.repetition, Int32(0), current.children) + parent = current.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + parent === nothing && return (completed, nextindex) + current = parent::_SchemaParseFrame + push!(current.children, completed) + current.completed += 1 + continue + end + child, childframe, nextindex = _schemaparsestart(elements, nextindex, + current.path, current.definition, current.repetition, + length(current.path) + 2, leaves, limits, budget, current) + if childframe === nothing + push!(current.children, child::SchemaNode) + current.completed += 1 + else + current = childframe::_SchemaParseFrame + end + end end -bit_or_byte_length(sch::Schema, schname::Vector{String}) = bit_or_byte_length(elem(sch, schname)) -bit_or_byte_length(schelem::SchemaElement) = hasproperty(schelem, :type_length) ? schelem.type_length : 0 - -num_children(schelem::SchemaElement) = hasproperty(schelem, :num_children) ? schelem.num_children : 0 - -function max_repetition_level(sch::Schema, schname::T) where {T <: AbstractVector{String}} - lev = isrepeated(sch, schname) ? 1 : 0 - istoplevel(schname) ? lev : (lev + max_repetition_level(sch, parentname(schname))) -end - -function max_definition_level(sch::Schema, schname::T) where {T <: AbstractVector{String}} - lev = isrequired(sch, schname) ? 0 : 1 - istoplevel(schname) ? lev : (lev + max_definition_level(sch, parentname(schname))) +function Schema(elements::Vector{Metadata.SchemaElement}; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + start = _budgetused(budget) + try + isempty(elements) && throw(FormatError("file metadata has an empty schema")) + elements[1].type_ === nothing || throw(FormatError("schema root must be a group")) + rootrepetition = elements[1].repetition_type + (rootrepetition === nothing || + rootrepetition == Metadata.FieldRepetitionType.REQUIRED) || + throw(FormatError("schema root can only use the legacy REQUIRED marker")) + rootchildren = _validateschemashape(elements[1], limits) + rootchildren <= length(elements) - 1 || throw(FormatError( + "group schema element $(repr(elements[1].name)) declares more direct " * + "children than remain in the flattened schema")) + iszero(rootchildren) && length(elements) > 1 && throw(FormatError( + "flattened schema has unclaimed elements")) + _checklimit(:container_elements, length(elements), limits.max_container_elements) + _reservearray!(budget, SchemaNode, length(elements)) + leaves = SchemaNode[] + sizehint!(leaves, length(elements)) + _reservearray!(budget, String, 0) + rootpath = String[] + root, nextindex = _parseschema(elements, leaves, rootpath, limits, budget) + nextindex == length(elements) + 1 || throw(FormatError( + "flattened schema has unclaimed elements")) + _reserveobjects!(budget) + return Schema(root, leaves) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end end -tables_schema(parfile) = tables_schema(schema(parfile)) -function tables_schema(sch::Schema) - cols = Parquet.ntcolstype(sch, sch.schema[1]) - colnames = fieldnames(cols) - coltypes = eltype.(fieldtypes(cols)) - Tables.Schema(colnames, coltypes) +function Schema(metadata::Metadata.FileMetaData; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + return Schema(metadata.schema; limits=limits, budget=budget) end - -logical_decimal_unscaled_type(precision::Int32) = (precision < 5) ? UInt16 : - (precision < 10) ? UInt32 : - (precision < 19) ? UInt64 : UInt128 - -function map_logical_decimal(precision::Int32, scale::Int32; use_float::Bool=false) - T = logical_decimal_unscaled_type(precision) - if scale == 0 - # integral values - return (signed(T), (bytes)->logical_decimal_integer(bytes, T)) - elseif use_float - # use Float64 - return (Float64, (bytes)->logical_decimal_float64(bytes, T, scale)) - else - # use Decimal - return (Decimal, (bytes)->logical_decimal_scaled(bytes, T, scale)) - end -end \ No newline at end of file diff --git a/src/show.jl b/src/show.jl deleted file mode 100644 index d9a828d..0000000 --- a/src/show.jl +++ /dev/null @@ -1,222 +0,0 @@ -function print_indent(io, n) - for d in 1:n - print(io, " ") - end -end - -function show(io::IO, cursor::RecordCursor) - par = cursor.par - rows = cursor.colcursors[1].rows - println(io, "Record Cursor on $(par.path)") - println(io, " rows: $rows") - - colpaths = [join(colname, '.') for colname in cursor.colnames] - println(io, " cols: $(join(colpaths, ", "))") -end - -function show(io::IO, cursor::BatchedColumnsCursor) - par = cursor.par - rows = cursor.colcursors[1].rows - println(io, "Batched Columns Cursor on $(par.path)") - println(io, " rows: $rows") - println(io, " batches: $(length(cursor))") - - colpaths = [join(colname, '.') for colname in cursor.colnames] - println(io, " cols: $(join(colpaths, ", "))") -end - -function show(io::IO, schema::SchemaElement, indent::AbstractString="", nchildren::Vector{Int}=Int[]) - print(io, indent) - lchildren = length(nchildren) - print_indent(io, lchildren) - if hasproperty(schema, :repetition_type) - r = schema.repetition_type - print(io, (r == FieldRepetitionType.REQUIRED) ? "required" : (r == FieldRepetitionType.OPTIONAL) ? "optional" : "repeated", " "); - end - hasproperty(schema, :_type) && print(io, Thrift.enumstr(_Type, schema._type), " ") - - print(io, schema.name) - hasproperty(schema, :field_id) && print(io, " (", schema.field_id, ")") - - if hasproperty(schema, :converted_type) - print(io, " # (from ", Thrift.enumstr(ConvertedType, schema.converted_type)) - if schema.converted_type == ConvertedType.DECIMAL - print(io, "(", schema.precision, ",", schema.scale, ")") - end - print(io, ") ") - end - - if hasproperty(schema, :num_children) && (getproperty(schema, :num_children) > 0) - push!(nchildren, schema.num_children) - print(io, " {") - elseif lchildren > 0 - nchildren[lchildren] -= 1 - if nchildren[lchildren] == 0 - pop!(nchildren) - println(io, "") - print_indent(io, length(nchildren)) - print(io, indent, "}") - end - end - - println(io, "") -end - -function show(io::IO, schema::Vector{SchemaElement}, indent::AbstractString="") - println(io, indent, "Schema:") - nchildren=Int[] - for schemaelem in schema - show(io, schemaelem, indent * " ", nchildren) - end -end - -show(io::IO, schema::Schema, indent::AbstractString="") = show(io, schema.schema, indent) - -function show(io::IO, kvmeta::KeyValue, indent::AbstractString="") - println(io, indent, kvmeta.key, " => ", kvmeta.value) -end - -function show(io::IO, kvmetas::Vector{KeyValue}, indent::AbstractString="") - isempty(kvmetas) && return - println(io, indent, "Metadata:") - for kvmeta in kvmetas - show(io, kvmeta, indent * " ") - end -end - -function show_encodings(io::IO, encodings::Vector{Int32}, indent::AbstractString="") - isempty(encodings) && return - print(io, indent, "Encodings: ") - pfx = "" - for encoding in encodings - print(io, pfx, Thrift.enumstr(Encoding, encoding)) - pfx = ", " - end - println(io, "") -end - -show(io::IO, hdr::IndexPageHeader, indent::AbstractString="") = nothing -function show(io::IO, page::DictionaryPageHeader, indent::AbstractString="") - println(io, indent, page.num_values, " values") -end - -function show(io::IO, hdr::DataPageHeader, indent::AbstractString="") - println(io, indent, hdr.num_values, " values") - println(io, indent, "encodings: values as ", Thrift.enumstr(Encoding, hdr.encoding), ", definitions as ", Thrift.enumstr(Encoding, hdr.definition_level_encoding), ", repetitions as ", Thrift.enumstr(Encoding, hdr.repetition_level_encoding)) - hasproperty(hdr, :statistics) && show(io, hdr.statistics, indent) -end - -function show(io::IO, hdr::DataPageHeaderV2, indent::AbstractString="") - compressed = hasproperty(hdr, :is_compressed) ? hdr.is_compressed : true - println(io, indent, hdr.num_values, " values, ", hdr.num_nulls, " nulls, ", hdr.num_rows, " rows, compressed:", compressed) - println(io, indent, "encoding:", Thrift.enumstr(Encoding, hdr.encoding), ", definition:", Thrift.enumstr(Encoding, hdr.definition_level_encoding), ", repetition:", Thrift.enumstr(Encoding, hdr.repetition_level_encoding)) - hasproperty(hdr, :statistics) && show(io, hdr.statistics, indent) -end - -function show(io::IO, page::PageHeader, indent::AbstractString="") - println(io, indent, Thrift.enumstr(PageType, page._type), " compressed bytes:", page.compressed_page_size, " (", page.uncompressed_page_size, " uncompressed)") - hasproperty(page, :data_page_header) && show(io, page.data_page_header, indent * " ") - hasproperty(page, :data_page_header_v2) && show(io, page.data_page_header_v2, indent * " ") - hasproperty(page, :index_page_header) && show(io, page.index_page_header, indent * " ") - hasproperty(page, :dictionary_page_header) && show(io, page.dictionary_page_header, indent * " ") -end - -function show(io::IO, pages::Vector{PageHeader}, indent::AbstractString="") - println(io, indent, "Pages:") - for page in pages - show(io, page, indent * " ") - end -end - -show(io::IO, page::Page, indent::AbstractString="") = show(io, page.hdr, indent) -show(io::IO, pages::Vector{Page}, indent::AbstractString="") = show(io, [page.hdr for page in pages], indent) - -function show(io::IO, stat::Statistics, indent::AbstractString="") - println(io, indent, "Statistics:") - if hasproperty(stat, :min) && hasproperty(stat, :max) - println(io, indent, " range:", stat.min, ":", stat.max) - elseif hasproperty(stat, :min) - println(io, indent, " min:", stat.min) - elseif hasproperty(stat, :max) - println(io, indent, " max:", stat.max) - end - hasproperty(stat, :null_count) && println(io, indent, " null count:", stat.null_count) - hasproperty(stat, :distinct_count) && println(io, indent, " distinct count:", stat.distinct_count) -end - -function show(io::IO, page_enc::PageEncodingStats, indent::AbstractString="") - println(io, indent, page_enc.count, " ", Thrift.enumstr(Encoding, page_enc.encoding), " encoded ", Thrift.enumstr(PageType, page_enc.page_type), " pages") -end - -function show(io::IO, page_encs::Vector{PageEncodingStats}, indent::AbstractString="") - isempty(page_encs) && return - println(io, indent, "Page encoding statistics:") - for page_enc in page_encs - show(io, page_enc, indent * " ") - end -end - -function show(io::IO, colmeta::ColumnMetaData, indent::AbstractString="") - println(io, indent, Thrift.enumstr(_Type, coltype(colmeta)), " ", join(colname(colmeta), '.'), ", num values:", colmeta.num_values) - show_encodings(io, colmeta.encodings, indent) - if colmeta.codec != CompressionCodec.UNCOMPRESSED - println(io, indent, Thrift.enumstr(CompressionCodec, colmeta.codec), " compressed bytes:", colmeta.total_compressed_size, " (", colmeta.total_uncompressed_size, " uncompressed)") - else - println(io, indent, Thrift.enumstr(CompressionCodec, colmeta.codec), " bytes:", colmeta.total_compressed_size) - end - - print(io, indent, "offsets: data:", colmeta.data_page_offset) - hasproperty(colmeta, :index_page_offset) && print(io, ", index:", colmeta.index_page_offset) - hasproperty(colmeta, :dictionary_page_offset) && print(io, ", dictionary:", colmeta.dictionary_page_offset) - println(io, "") - hasproperty(colmeta, :statistics) && show(io, colmeta.statistics, indent) - hasproperty(colmeta, :encoding_stats) && show(io, colmeta.encoding_stats, indent) - hasproperty(colmeta, :key_value_metadata) && show(io, colmeta.key_value_metadata, indent) -end - -function show(io::IO, columns::Vector{ColumnChunk}, indent::AbstractString="") - for col in columns - path = hasproperty(col, :file_path) ? col.file_path : "" - println(io, indent, "Column at offset: ", path, "#", col.file_offset) - show(io, col.meta_data, indent * " ") - end -end - -function show(io::IO, grp::RowGroup, indent::AbstractString="") - println(io, indent, "Row Group: ", grp.num_rows, " rows in ", grp.total_byte_size, " bytes") - show(io, grp.columns, indent * " ") -end - -function show(io::IO, row_groups::Vector{RowGroup}, indent::AbstractString="") - println(io, indent, "Row Groups:") - for grp in row_groups - show(io, grp, indent * " ") - end -end - -function show(io::IO, meta::FileMetaData, indent::AbstractString="") - println(io, indent, "version: ", meta.version) - println(io, indent, "nrows: ", meta.num_rows) - println(io, indent, "created by: ", meta.created_by) - - show(io, meta.schema, indent) - show(io, meta.row_groups, indent) - hasproperty(meta, :key_value_metadata) && show(io, meta.key_value_metadata, indent) -end - -function show(io::IO, par::Parquet.File) - println(io, "Parquet file: $(par.path)") - meta = par.meta - println(io, " version: $(meta.version)") - println(io, " nrows: $(meta.num_rows)") - println(io, " created by: $(meta.created_by)") - println(io, " cached: $(length(par.page_cache.refs)) column chunks") -end - -function show(io::IO, table::Parquet.Table) - print(io, "Parquet.Table(\"$(getfield(table, :path))\")") -end - -function show(io::IO, dataset::Parquet.Dataset) - print(io, "Parquet.Dataset(\"$(getfield(dataset, :path))\")") -end diff --git a/src/simple_reader.jl b/src/simple_reader.jl deleted file mode 100644 index 68eaf04..0000000 --- a/src/simple_reader.jl +++ /dev/null @@ -1,197 +0,0 @@ -""" - read_parquet(path; kwargs...) - -Returns the table contained in the parquet file or dataset (partitioned parquet files in a folder) -in a Tables.jl compatible format. - -Options: -- `rows`: The row range to iterate through, all rows by default. Applicable only when reading a single file. -- `filter`: Filter function to apply while loading only a subset of partitions from a dataset. -- `batchsize`: Maximum number of rows to read in each batch (default: row count of first row group). Applied only when reading a single file, and to each file when reading a dataset. -- `use_threads`: Whether to use threads while reading the file; applicable only for Julia v1.3 and later and switched on by default if julia processes is started with multiple threads. -- `column_generator`: Function to generate a partitioned column when not found in the partitioned table. Parameters provided to the function: table, column index, length of column to generate. Default implementation determines column values from the table path. - -One can easily convert the returned object to any Tables.jl compatible table e.g. DataFrames.DataFrame via - -``` -using DataFrames -df = DataFrame(read_parquet(path)) -``` -""" -function read_parquet(path; kwargs...) - if isdir(path) - Parquet.Dataset(path; kwargs...) - elseif isfile(path) - Parquet.Table(path; kwargs...) - else - error("Invalid path to parquet file or dataset - $path") - end -end - -""" - Parquet.Table(path; kwargs...) - -Returns the table contained in the parquet file in a Tables.jl compatible format. - -Options: -- `rows`: The row range to iterate through, all rows by default. -- `batchsize`: Maximum number of rows to read in each batch (default: row count of first row group). -- `use_threads`: Whether to use threads while reading the file; applicable only for Julia v1.3 and later and switched on by default if julia processes is started with multiple threads. - -One can easily convert the returned object to any Tables.jl compatible table e.g. DataFrames.DataFrame via - -``` -using DataFrames -df = DataFrame(read_parquet(path)) -``` -""" -struct Table <: Tables.AbstractColumns - path::String - ncols::Int - rows::Union{Nothing,UnitRange} - batchsize::Union{Nothing,Signed} - use_threads::Bool - parfile::Parquet.File - schema::Tables.Schema - lookup::Dict{Symbol, Int} # map column name => index - columns::Vector{AbstractVector} - column_generator::Function - - Table(path, sch::Tables.Schema; kwargs...) = Table(path, Parquet.File(path), sch; kwargs...) - function Table(path, parfile::Parquet.File=Parquet.File(path), sch::Tables.Schema=tables_schema(parfile); - rows::Union{Nothing,UnitRange}=nothing, - batchsize::Union{Nothing,Signed}=nothing, - column_generator::Function=column_generator, - use_threads::Bool=(nthreads() > 1)) - ncols = length(sch.names) - lookup = Dict{Symbol, Int}(nm => i for (i, nm) in enumerate(sch.names)) - new(path, ncols, rows, batchsize, use_threads, parfile, sch, lookup, AbstractVector[], column_generator) - end -end - -function close(table::Table) - empty!(getfield(table, :columns)) - close(getfield(table, :parfile)) -end - -function column_generator(table::Table, colidx::Int, len::Int) - schema = getfield(table, :schema) - coltype = schema.types[colidx] - missingval = nonmissingtype(coltype) === coltype ? undef : missing - Array{coltype}(missingval, len) -end - -""" -Represents one partition of the parquet file. -Typically a row group, but could be any other unit as mentioned while opening the table. -""" -struct TablePartition <: Tables.AbstractColumns - table::Table - columns::Vector{AbstractVector} -end - -""" -Iterator to iterate over partitions of a parquet file, returned by the `Tables.partitions(table)` method. -Each partition is typically a row group, but could be any other unit as mentioned while opening the table. -""" -struct TablePartitions - table::Table - ncols::Int - schema::Tables.Schema - cursor::BatchedColumnsCursor - - function TablePartitions(table::Table) - new(table, getfield(table, :ncols), getfield(table, :schema), cursor(table)) - end -end -length(tp::TablePartitions) = length(tp.cursor) -function iterated_partition(partitions::TablePartitions, iterresult) - (iterresult === nothing) && (return nothing) - chunk, batchid = iterresult - columns = AbstractVector[] - for colidx in 1:partitions.ncols - colname = partitions.schema.names[colidx] - if hasproperty(chunk, colname) - push!(columns, getproperty(chunk, colname)) - else - generator = getfield(partitions.table, :column_generator) - push!(columns, generator(partitions.table, colidx, length(first(chunk)))) - end - end - TablePartition(partitions.table, columns), batchid -end -Base.iterate(partitions::TablePartitions, batchid) = iterated_partition(partitions, iterate(partitions.cursor, batchid)) -Base.iterate(partitions::TablePartitions) = iterated_partition(partitions, iterate(partitions.cursor)) - -function cursor(table::Table) - kwargs = Dict{Symbol,Any}(:use_threads => getfield(table, :use_threads), :reusebuffer => false) - (getfield(table, :rows) === nothing) || (kwargs[:rows] = getfield(table, :rows)) - (getfield(table, :batchsize) === nothing) || (kwargs[:batchsize] = getfield(table, :batchsize)) - BatchedColumnsCursor(getfield(table, :parfile); kwargs...) -end - -loaded(table::Table) = !isempty(getfield(table, :columns)) -load(table::Table) = load(table, cursor(table)) -function load(table::Table, colcursor::BatchedColumnsCursor) - chunks = [chunk for chunk in colcursor] - ncols = getfield(table, :ncols) - columns = getfield(table, :columns) - schema = getfield(table, :schema) - generator = getfield(table, :column_generator) - - empty!(columns) - nchunks = length(chunks) - if nchunks == 1 - chunk = chunks[1] - for colidx in 1:ncols - colname = schema.names[colidx] - if hasproperty(chunk, colname) - push!(columns, getproperty(chunk, colname)) - else - push!(columns, generator(table, colidx, length(first(chunk)))) - end - end - elseif nchunks > 1 - for colidx in 1:ncols - colname = schema.names[colidx] - coltype = schema.types[colidx] - vecs = Vector{coltype}[] - for chunkidx in 1:nchunks - chunk = chunks[chunkidx] - if hasproperty(chunk, colname) - push!(vecs, getproperty(chunk, colname)) - else - push!(vecs, generator(table, colidx, length(first(chunk)))) - end - end - push!(columns, ChainedVector(vecs)) - end - else - schema = getfield(table, :schema) - coltypes = schema.types - for colidx in 1:ncols - push!(columns, (coltypes[colidx])[]) - end - end - nothing -end - -Tables.istable(::Table) = true -Tables.columnaccess(::Table) = true -Tables.schema(t::Table) = getfield(t, :schema) -Tables.columnnames(t::Table) = getfield(t, :schema).names -Tables.columns(t::Table) = Tables.CopiedColumns(t) -Tables.getcolumn(t::Table, nm::Symbol) = Tables.getcolumn(t, getfield(t, :lookup)[nm]) -function Tables.getcolumn(t::Table, i::Int) - loaded(t) || load(t) - getfield(t, :columns)[i] -end -Tables.partitions(t::Table) = TablePartitions(t) - -Tables.istable(::TablePartition) = true -Tables.columnaccess(::TablePartition) = true -Tables.schema(tp::TablePartition) = Tables.schema(getfield(tp, :table)) -Tables.columnnames(tp::TablePartition) = Tables.columnnames(getfield(tp, :table)) -Tables.columns(tp::TablePartition) = Tables.CopiedColumns(tp) -Tables.getcolumn(tp::TablePartition, nm::Symbol) = Tables.getcolumn(tp, getfield(getfield(tp, :table), :lookup)[nm]) -Tables.getcolumn(tp::TablePartition, i::Int) = getfield(tp, :columns)[i] diff --git a/src/source.jl b/src/source.jl new file mode 100644 index 0000000..1b2e216 --- /dev/null +++ b/src/source.jl @@ -0,0 +1,232 @@ +abstract type AbstractSource end + +function close!(::AbstractSource) + return +end + +function concurrentreads(::AbstractSource) + return false +end + +mutable struct OwnerRegion{B<:AbstractVector{UInt8},I} + bytes::B + io::I + budget::Union{Nothing,_LiveByteBudget} + @atomic materializedcharge::Int64 + @atomic closed::Bool +end + +struct MemorySource{R<:OwnerRegion} <: AbstractSource + region::R +end + +struct BufferSlice{R<:OwnerRegion} <: AbstractVector{UInt8} + region::R + offset::Int64 + count::Int64 + function BufferSlice(region::R, offset::Int64, count::Int64) where {R<:OwnerRegion} + offset >= 0 || throw(BoundsError(region.bytes, offset)) + count >= 0 || throw(ArgumentError("byte count must be nonnegative")) + last = try + Base.checked_add(offset, count) + catch err + err isa OverflowError || rethrow() + throw(BoundsError(region.bytes, (offset, count))) + end + last <= length(region.bytes) || throw(BoundsError(region.bytes, (offset, count))) + return new{R}(region, offset, count) + end +end + +function BufferSlice(region::OwnerRegion, offset::Integer, count::Integer) + return BufferSlice(region, Int64(offset), Int64(count)) +end + +function Base.IndexStyle(::Type{<:BufferSlice}) + return IndexLinear() +end + +function Base.size(bytes::BufferSlice) + return (Int(bytes.count),) +end + +function Base.length(bytes::BufferSlice) + return Int(bytes.count) +end + +function Base.getindex(bytes::BufferSlice, index::Int) + (@atomic bytes.region.closed) && throw(ArgumentError("Parquet byte region is closed")) + checkbounds(bytes, index) + first = firstindex(bytes.region.bytes) + return bytes.region.bytes[first + Int(bytes.offset) + index - 1] +end + +function Base.copy(bytes::BufferSlice) + (@atomic bytes.region.closed) && throw(ArgumentError("Parquet byte region is closed")) + return collect(bytes) +end + +function close!(region::OwnerRegion) + (@atomicswap region.closed = true) && return + charge = @atomicswap region.materializedcharge = Int64(0) + try + region.io === nothing || close(region.io) + finally + iszero(charge) || _release!(something(region.budget), charge) + end + return +end + +function close!(source::MemorySource) + close!(source.region) + return +end + +function Base.close(source::MemorySource) + close!(source) + return +end + +function source(bytes::AbstractVector{UInt8}; + budget::_LiveByteBudget=_LiveByteBudget(Limits())) + region = OwnerRegion(bytes, nothing, nothing, Int64(0), false) + return MemorySource(region) +end + +function _readsourcebytes(io::IO, budget::_LiveByteBudget) + temporary = Int64(0) + outputcharge = Int64(0) + try + temporary = _materializedsum(temporary, + _reservearray!(budget, Vector{UInt8}, 0)) + chunks = Vector{UInt8}[] + available = budget.maximum - _budgetused(budget) + fixed = 3 * _MATERIALIZED_ARRAY_HEADER_BYTES + 2 * Int64(sizeof(Ptr{Cvoid})) + blocksize = Int(min(Int64(64 * 1024), max(Int64(1), + (available - fixed) ÷ 3))) + temporary = _materializedsum(temporary, + _reservearray!(budget, UInt8, blocksize)) + scratch = Vector{UInt8}(undef, blocksize) + total = Int64(0) + while !eof(io) + count = readbytes!(io, scratch, blocksize) + iszero(count) && continue + total = try + Base.checked_add(total, Int64(count)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:materialized_bytes, typemax(Int64), + budget.maximum)) + end + charge = _reservearray!(budget, UInt8, count) + temporary = _materializedsum(temporary, charge) + chunk = Vector{UInt8}(undef, count) + copyto!(chunk, 1, scratch, 1, count) + pointercharge = _reservearray!(budget, Vector{UInt8}, 2; + header=false) + temporary = _materializedsum(temporary, pointercharge) + push!(chunks, chunk) + end + total <= typemax(Int) || throw(LimitError(:materialized_bytes, + typemax(Int64), budget.maximum)) + outputcharge = _reservearray!(budget, UInt8, total) + bytes = Vector{UInt8}(undef, Int(total)) + position = 1 + for chunk in chunks + copyto!(bytes, position, chunk, 1, length(chunk)) + position += length(chunk) + end + _release!(budget, temporary) + return bytes, outputcharge + catch + iszero(outputcharge) || _release!(budget, outputcharge) + iszero(temporary) || _release!(budget, temporary) + rethrow() + end +end + +function source(io::IO; budget::_LiveByteBudget=_LiveByteBudget(Limits())) + bytes, charge = _readsourcebytes(io, budget) + try + region = OwnerRegion(bytes, nothing, budget, charge, false) + return MemorySource(region) + catch + _release!(budget, charge) + rethrow() + end +end + +function source(path::AbstractString; + budget::_LiveByteBudget=_LiveByteBudget(Limits())) + io = open(path, "r") + try + count = filesize(io) + bytes = iszero(count) ? UInt8[] : Mmap.mmap(io, Vector{UInt8}, count) + region = OwnerRegion(bytes, io, nothing, Int64(0), false) + return MemorySource(region) + catch + try + close(io) + catch + end + rethrow() + end +end + +function source(src::AbstractSource; + budget::_LiveByteBudget=_LiveByteBudget(Limits())) + return src +end + +function sourcelength(src::MemorySource) + (@atomic src.region.closed) && throw(ArgumentError("Parquet byte region is closed")) + return Int64(length(src.region.bytes)) +end + +function concurrentreads(::MemorySource) + return true +end + +function readrange(src::MemorySource, offset::Integer, count::Integer) + (@atomic src.region.closed) && throw(ArgumentError("Parquet byte region is closed")) + return BufferSlice(src.region, offset, count) +end + +function _checkedsourcelength(src::AbstractSource) + total = sourcelength(src) + total isa Integer && !(total isa Bool) || throw(ArgumentError( + "source length callback must return an integer")) + total >= 0 || throw(ArgumentError( + "source length callback returned a negative length")) + total <= typemax(Int64) || throw(ArgumentError( + "source length callback returned a length that exceeds Int64")) + return Int64(total) +end + +function _readrangeexact(src::AbstractSource, total::Int64, offset::Int64, + count::Int64) + total >= 0 || throw(ArgumentError( + "authoritative source length must be nonnegative")) + offset >= 0 || throw(ArgumentError("source read offset must be nonnegative")) + count >= 0 || throw(ArgumentError("source read count must be nonnegative")) + stop = try + Base.checked_add(offset, count) + catch err + err isa OverflowError || rethrow() + throw(ArgumentError("source read range overflows Int64")) + end + stop <= total || throw(ArgumentError( + "source read range extends past the authoritative source length")) + count <= typemax(Int) || throw(ArgumentError( + "source read count does not fit Int")) + bytes = readrange(src, offset, count) + bytes isa AbstractVector{UInt8} || throw(ArgumentError( + "source read callback must return AbstractVector{UInt8}")) + length(bytes) == count || throw(ArgumentError( + "source read callback returned the wrong byte count")) + byteaxes = axes(bytes) + length(byteaxes) == 1 && byteaxes[1] isa Base.OneTo && + byteaxes[1] == Base.OneTo(Int(count)) || throw(ArgumentError( + "source read callback must return a vector with Base.OneTo axes")) + return bytes +end diff --git a/src/statistics.jl b/src/statistics.jl new file mode 100644 index 0000000..caf4a7b --- /dev/null +++ b/src/statistics.jl @@ -0,0 +1,1116 @@ +struct _StatisticOrderFact + declared::Symbol + comparison::Symbol +end + +struct _StatisticBoundFact + state::Symbol + raw::Union{Nothing,Vector{UInt8}} + exactness::Symbol + reason::Symbol + adjustment::Symbol +end + +struct _StatisticCountFact + state::Symbol + value::Union{Nothing,Int64} +end + +struct _StatisticTrustFact + state::Symbol + reason::Symbol +end + +struct _ColumnStatisticFacts + lower::_StatisticBoundFact + upper::_StatisticBoundFact + null_count::_StatisticCountFact + nan_count::_StatisticCountFact + distinct_count::_StatisticCountFact + order::_StatisticOrderFact + trust::_StatisticTrustFact + family::Symbol + comparison::Symbol + occupancy::Symbol +end + +struct _StatisticLeafSemantics + comparison::Symbol + logical::Any + floating::Bool +end + +struct _StatisticProducerVersion + present::Bool + major::UInt64 + minor::UInt64 + patch::UInt64 + prerelease::Bool + cdh_exception::Bool +end + +struct _StatisticProducer + application::Symbol + parsed::Bool + version::_StatisticProducerVersion +end + +const _NO_STATISTIC_VERSION = _StatisticProducerVersion(false, 0, 0, 0, false, + false) + +function _statisticlimit(limits::Limits) + limit = limits.max_statistics_value_bytes + limit >= 0 || throw(ArgumentError( + "max_statistics_value_bytes must be nonnegative, got $limit")) + return limit +end + +function _leafstatisticsemantics(element::Metadata.SchemaElement) + logical = _logicalkind(element) + floating = element.type_ in (Metadata.Type.FLOAT, Metadata.Type.DOUBLE) || + logical === :float16 + if logical isa _IntegerLogicalKind + comparison = logical.signed ? :signed : :unsigned + return _StatisticLeafSemantics(comparison, logical, floating) + elseif logical isa Union{_TimeLogicalKind,_TimestampLogicalKind} + return _StatisticLeafSemantics(:signed, logical, floating) + elseif logical in (:string, :enum, :uuid, :json, :bson) + return _StatisticLeafSemantics(:unsigned_bytes, logical, floating) + elseif logical === :date + return _StatisticLeafSemantics(:signed, logical, floating) + elseif logical === :decimal + return _StatisticLeafSemantics(:decimal, logical, floating) + elseif logical === :float16 + return _StatisticLeafSemantics(:floating, logical, floating) + elseif logical in (:interval, :unknown) + return _StatisticLeafSemantics(:undefined, logical, floating) + elseif logical !== nothing + return _StatisticLeafSemantics(:undefined, logical, floating) + end + annotated = element.logicalType !== nothing || element.converted_type !== nothing + annotated && return _StatisticLeafSemantics(:undefined, nothing, floating) + physical = element.type_ + physical == Metadata.Type.BOOLEAN && + return _StatisticLeafSemantics(:boolean, nothing, floating) + physical in (Metadata.Type.INT32, Metadata.Type.INT64) && + return _StatisticLeafSemantics(:signed, nothing, floating) + physical in (Metadata.Type.FLOAT, Metadata.Type.DOUBLE) && + return _StatisticLeafSemantics(:floating, nothing, floating) + physical in (Metadata.Type.BYTE_ARRAY, Metadata.Type.FIXED_LEN_BYTE_ARRAY) && + return _StatisticLeafSemantics(:unsigned_bytes, nothing, floating) + return _StatisticLeafSemantics(:undefined, nothing, floating) +end + +function _columnorderdeclaration(order::Metadata.ColumnOrder) + order.TYPE_ORDER !== nothing && return :type_order + order.IEEE_754_TOTAL_ORDER !== nothing && return :ieee_total_order + isempty(order.unknown_fields) && + throw(FormatError("ColumnOrder union has no member")) + length(order.unknown_fields) == 1 || + throw(FormatError("ColumnOrder union has more than one unknown member")) + return :unknown +end + +function _validatecolumnorders(schema::Schema, leafindex::Int, + orders::Union{Nothing,AbstractVector{Metadata.ColumnOrder}}) + orders === nothing && return :absent + length(orders) == length(schema.leaves) || throw(FormatError( + "column_orders has $(length(orders)) entries for $(length(schema.leaves)) leaves")) + selected = :absent + for index in eachindex(schema.leaves) + semantics = _leafstatisticsemantics(schema.leaves[index].element) + declaration = _columnorderdeclaration(orders[index]) + declaration === :ieee_total_order && !semantics.floating && + throw(FormatError("IEEE_754_TOTAL_ORDER is invalid for non-floating leaf " * + "$(repr(schema.leaves[index].element.name))")) + index == leafindex && (selected = declaration) + end + return selected +end + +function _statisticorder(semantics::_StatisticLeafSemantics, declared::Symbol) + comparison = if declared === :type_order + semantics.comparison + elseif declared === :ieee_total_order + :ieee_total_order + else + :undefined + end + return _StatisticOrderFact(declared, comparison) +end + +function _absentcount() + return _StatisticCountFact(:absent, nothing) +end + +function _validatedcount(name::Symbol, value::Union{Nothing,Int64}, total::Int64) + value === nothing && return _absentcount() + 0 <= value <= total || throw(FormatError( + "$name $value is outside the valid range 0:$total")) + return _StatisticCountFact(:known, value) +end + +function _checkedcountsum(left::Int64, right::Int64, label::String) + return try + Base.checked_add(left, right) + catch err + err isa OverflowError || rethrow() + throw(FormatError("$label overflows Int64")) + end +end + +function _validatestatisticcounts(statistics::Union{Nothing,Metadata.Statistics}, + total::Int64, floating::Bool) + statistics === nothing && return (_absentcount(), _absentcount(), _absentcount()) + nulls = _validatedcount(:null_count, statistics.null_count, total) + nans = _validatedcount(:nan_count, statistics.nan_count, total) + distinct = _validatedcount(:distinct_count, statistics.distinct_count, total) + nans.state === :known && !floating && throw(FormatError( + "nan_count is only valid for FLOAT, DOUBLE, and FLOAT16 leaves")) + _validatecountrelationships(nulls, nans, distinct, total) + return nulls, nans, distinct +end + +function _validatecountrelationships(nulls::_StatisticCountFact, + nans::_StatisticCountFact, distinct::_StatisticCountFact, total::Int64) + if nulls.state === :known && nans.state === :known + combined = _checkedcountsum(nulls.value::Int64, nans.value::Int64, + "null_count + nan_count") + combined <= total || throw(FormatError( + "null_count + nan_count exceeds num_values $total")) + end + if nulls.state === :known && distinct.state === :known + available = total - (nulls.value::Int64) + (distinct.value::Int64) <= available || throw(FormatError( + "distinct_count exceeds non-null value count $available")) + end + return +end + +function _statisticoccupancy(nulls::_StatisticCountFact, + nans::_StatisticCountFact, total::Int64, floating::Bool) + iszero(total) && return :no_non_null + if nulls.state === :known && nulls.value == total + return :no_non_null + end + floating || return :unknown + nulls.state === :known && nans.state === :known || return :unknown + nonnull = total - (nulls.value::Int64) + nonnull > 0 || return :no_non_null + nancount = nans.value::Int64 + nancount == nonnull && return :all_nan + nancount < nonnull && return :has_non_nan + throw(AssertionError("validated floating counts have an impossible occupancy")) +end + +function _readstatisticuint16(bytes::AbstractVector{UInt8}) + return UInt16(bytes[1]) | (UInt16(bytes[2]) << 8) +end + +function _readstatisticuint32(bytes::AbstractVector{UInt8}) + return UInt32(bytes[1]) | (UInt32(bytes[2]) << 8) | + (UInt32(bytes[3]) << 16) | (UInt32(bytes[4]) << 24) +end + +function _readstatisticuint64(bytes::AbstractVector{UInt8}) + value = UInt64(0) + for index in 8:-1:1 + value = (value << 8) | UInt64(bytes[index]) + end + return value +end + +function _statisticplainwidth(element::Metadata.SchemaElement) + physical = element.type_ + physical == Metadata.Type.BOOLEAN && return 1 + physical in (Metadata.Type.INT32, Metadata.Type.FLOAT) && return 4 + physical in (Metadata.Type.INT64, Metadata.Type.DOUBLE) && return 8 + physical == Metadata.Type.INT96 && return 12 + physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY && + return Int(something(element.type_length)) + return nothing +end + +function _checkstatisticwidth(element::Metadata.SchemaElement, + raw::Vector{UInt8}) + width = _statisticplainwidth(element) + width === nothing && return + length(raw) == width || throw(FormatError( + "statistics bound for $(repr(element.name)) has $(length(raw)) bytes, expected $width")) + return +end + +function _statisticexactness(family::Symbol, raw, flag::Union{Nothing,Bool}) + raw === nothing && return :unknown + family === :deprecated && return :unknown + flag === nothing && return :unknown + return flag ? :exact : :inexact +end + +function _absentbound() + return _StatisticBoundFact(:absent, nothing, :unknown, :absent, :none) +end + +function _unknownbound(bound::_StatisticBoundFact, reason::Symbol) + bound.state === :absent && return bound + bound.state === :unknown && return bound + return _StatisticBoundFact(:unknown, bound.raw, bound.exactness, reason, :none) +end + +function _preparestatisticbound(element::Metadata.SchemaElement, raw, + family::Symbol, flag::Union{Nothing,Bool}, limit::Int64) + raw === nothing && return _absentbound() + _checkstatisticwidth(element, raw) + exactness = _statisticexactness(family, raw, flag) + bound = _StatisticBoundFact(:known, raw, exactness, :valid, :none) + length(raw) <= limit && return bound + return _unknownbound(bound, :over_limit) +end + +function _statisticsignedvalue(element::Metadata.SchemaElement, + raw::AbstractVector{UInt8}) + element.type_ == Metadata.Type.INT32 && + return Int64(reinterpret(Int32, _readstatisticuint32(raw))) + element.type_ == Metadata.Type.INT64 && + return reinterpret(Int64, _readstatisticuint64(raw)) + throw(ArgumentError("signed statistics comparison requires INT32 or INT64")) +end + +function _statisticunsignedvalue(element::Metadata.SchemaElement, + raw::AbstractVector{UInt8}) + element.type_ == Metadata.Type.INT32 && return UInt64(_readstatisticuint32(raw)) + element.type_ == Metadata.Type.INT64 && return _readstatisticuint64(raw) + throw(ArgumentError("unsigned statistics comparison requires INT32 or INT64")) +end + +function _statisticintegerfits(kind::_IntegerLogicalKind, + element::Metadata.SchemaElement, raw::AbstractVector{UInt8}) + width = Int(kind.bitwidth) + if kind.signed + width == 64 && return true + value = _statisticsignedvalue(element, raw) + limit = Int64(1) << (width - 1) + return -limit <= value < limit + end + width == 64 && return true + value = _statisticunsignedvalue(element, raw) + limit = (UInt64(1) << width) - UInt64(1) + return value <= limit +end + +function _statistictimevalid(kind::_TimeLogicalKind, + element::Metadata.SchemaElement, raw::AbstractVector{UInt8}) + value = _statisticsignedvalue(element, raw) + limit, _ = _timeparameters(kind.unit) + return 0 <= value < limit +end + +function _unsignedmagnitude(value::Int64) + value >= 0 && return UInt64(value) + return UInt64(-(value + 1)) + UInt64(1) +end + +function _decimaldigitcount(value::Int64) + magnitude = _unsignedmagnitude(value) + magnitude == 0 && return 1 + digits = 0 + while magnitude != 0 + magnitude = div(magnitude, UInt64(10)) + digits += 1 + end + return digits +end + +function _decimalmagnitude!(output::Vector{UInt8}, raw::AbstractVector{UInt8}) + copyto!(output, raw) + iszero(first(raw) & 0x80) && return output + for index in eachindex(output) + @inbounds output[index] = ~output[index] + end + for index in lastindex(output):-1:firstindex(output) + @inbounds output[index] += UInt8(1) + @inbounds iszero(output[index]) || break + end + return output +end + +function _decimalchunkdigits(value::UInt64) + value == 0 && return 1 + digits = 0 + while value != 0 + value = div(value, UInt64(10)) + digits += 1 + end + return digits +end + +function _dividedecimalchunk!(magnitude::Vector{UInt8}, start::Int) + remainder = UInt64(0) + divisor = UInt64(1_000_000_000) + for index in start:lastindex(magnitude) + current = (remainder << 8) | UInt64(magnitude[index]) + magnitude[index] = UInt8(div(current, divisor)) + remainder = rem(current, divisor) + end + while start <= lastindex(magnitude) && iszero(magnitude[start]) + start += 1 + end + return start, remainder +end + +function _decimaldigitcount(raw::AbstractVector{UInt8}, budget::_LiveByteBudget) + isempty(raw) && return nothing + charge = _reservearray!(budget, UInt8, length(raw)) + try + magnitude = Vector{UInt8}(undef, length(raw)) + _decimalmagnitude!(magnitude, raw) + start = firstindex(magnitude) + while start <= lastindex(magnitude) && iszero(magnitude[start]) + start += 1 + end + start > lastindex(magnitude) && return 1 + chunks = 0 + leading = UInt64(0) + while start <= lastindex(magnitude) + start, leading = _dividedecimalchunk!(magnitude, start) + chunks += 1 + end + return (chunks - 1) * 9 + _decimalchunkdigits(leading) + finally + _release!(budget, charge) + end +end + +function _statisticdecimalvalid(element::Metadata.SchemaElement, + raw::AbstractVector{UInt8}, budget::_LiveByteBudget) + precision, _ = something(_decimalparameters(element)) + physical = element.type_ + digits = if physical in (Metadata.Type.INT32, Metadata.Type.INT64) + _decimaldigitcount(_statisticsignedvalue(element, raw)) + else + _decimaldigitcount(raw, budget) + end + digits === nothing && return false + return digits <= precision +end + +function _optionallogicaldocumentvalid(kind::Symbol, raw::AbstractVector{UInt8}, + limits::Limits, budget::_LiveByteBudget) + charge = _reserveobjects!(budget) + try + try + kind === :json && _validatejson(raw, limits, FormatError) + kind === :bson && _validatebson(raw, limits, FormatError) + return true + catch err + err isa Union{FormatError,LimitError} && return false + rethrow() + end + finally + _release!(budget, charge) + end +end + +function _statisticsemanticvalid(semantics::_StatisticLeafSemantics, + element::Metadata.SchemaElement, raw::AbstractVector{UInt8}, limits::Limits, + budget::_LiveByteBudget) + logical = semantics.logical + logical isa _IntegerLogicalKind && + return _statisticintegerfits(logical, element, raw) + logical isa _TimeLogicalKind && + return _statistictimevalid(logical, element, raw) + logical === :decimal && return _statisticdecimalvalid(element, raw, budget) + logical in (:string, :enum) && return isvalid(String, raw) + logical in (:json, :bson) && + return _optionallogicaldocumentvalid(logical, raw, limits, budget) + semantics.comparison === :boolean && return raw[1] in (0x00, 0x01) + return true +end + +function _validatestatisticbound(bound::_StatisticBoundFact, + semantics::_StatisticLeafSemantics, element::Metadata.SchemaElement, + limits::Limits, budget::_LiveByteBudget) + bound.state === :known || return bound + raw = bound.raw::Vector{UInt8} + _statisticsemanticvalid(semantics, element, raw, limits, budget) && return bound + return _unknownbound(bound, :invalid_value) +end + +function _comparestatisticbytes(left::AbstractVector{UInt8}, + right::AbstractVector{UInt8}) + for (leftbyte, rightbyte) in zip(left, right) + leftbyte < rightbyte && return Int8(-1) + leftbyte > rightbyte && return Int8(1) + end + length(left) < length(right) && return Int8(-1) + length(left) > length(right) && return Int8(1) + return Int8(0) +end + +function _normalizedsignedstart(bytes::AbstractVector{UInt8}) + start = firstindex(bytes) + stop = lastindex(bytes) + negative = !iszero(bytes[start] & 0x80) + extension = negative ? UInt8(0xff) : UInt8(0x00) + while start < stop && bytes[start] == extension + nextnegative = !iszero(bytes[start + 1] & 0x80) + nextnegative == negative || break + start += 1 + end + return start, negative +end + +function _comparestatisticdecimalbytes(left::AbstractVector{UInt8}, + right::AbstractVector{UInt8}) + leftstart, leftnegative = _normalizedsignedstart(left) + rightstart, rightnegative = _normalizedsignedstart(right) + leftnegative != rightnegative && return leftnegative ? Int8(-1) : Int8(1) + leftlength = lastindex(left) - leftstart + 1 + rightlength = lastindex(right) - rightstart + 1 + if leftlength != rightlength + order = leftlength < rightlength ? Int8(-1) : Int8(1) + return leftnegative ? -order : order + end + for offset in 0:(leftlength - 1) + leftbyte = left[leftstart + offset] + rightbyte = right[rightstart + offset] + leftbyte < rightbyte && return Int8(-1) + leftbyte > rightbyte && return Int8(1) + end + return Int8(0) +end + +function _comparestatisticnumbers(left, right) + left < right && return Int8(-1) + left > right && return Int8(1) + return Int8(0) +end + +function _statisticfloatbits(element::Metadata.SchemaElement, + raw::AbstractVector{UInt8}) + physical = element.type_ + physical == Metadata.Type.FLOAT && return _readstatisticuint32(raw) + physical == Metadata.Type.DOUBLE && return _readstatisticuint64(raw) + return _readstatisticuint16(raw) +end + +function _statisticfloatvalue(bits::UInt16) + return reinterpret(Float16, bits) +end + +function _statisticfloatvalue(bits::UInt32) + return reinterpret(Float32, bits) +end + +function _statisticfloatvalue(bits::UInt64) + return reinterpret(Float64, bits) +end + +function _statisticfloatmasks(bits::UInt16) + return UInt16(0x8000), UInt16(0x7c00), UInt16(0x03ff) +end + +function _statisticfloatmasks(bits::UInt32) + return UInt32(0x80000000), UInt32(0x7f800000), UInt32(0x007fffff) +end + +function _statisticfloatmasks(bits::UInt64) + return UInt64(0x8000000000000000), UInt64(0x7ff0000000000000), + UInt64(0x000fffffffffffff) +end + +function _statisticisnan(bits::Union{UInt16,UInt32,UInt64}) + _, exponent, fraction = _statisticfloatmasks(bits) + return bits & exponent == exponent && !iszero(bits & fraction) +end + +function _statisticiszero(bits::Union{UInt16,UInt32,UInt64}) + sign, _, _ = _statisticfloatmasks(bits) + return iszero(bits & ~sign) +end + +function _statisticisnegative(bits::Union{UInt16,UInt32,UInt64}) + sign, _, _ = _statisticfloatmasks(bits) + return !iszero(bits & sign) +end + +function _statisticieeekey(bits::T) where {T<:Union{UInt16,UInt32,UInt64}} + sign, _, _ = _statisticfloatmasks(bits) + return iszero(bits & sign) ? bits | sign : ~bits +end + +function _adjustedstatisticfloat(bits::T, adjustment::Symbol) where + {T<:Union{UInt16,UInt32,UInt64}} + sign, _, _ = _statisticfloatmasks(bits) + adjustment === :negative_zero && return _statisticfloatvalue(sign) + adjustment === :positive_zero && return _statisticfloatvalue(zero(T)) + return _statisticfloatvalue(bits) +end + +function _comparestatisticfloats(element::Metadata.SchemaElement, + left::AbstractVector{UInt8}, right::AbstractVector{UInt8}, comparison::Symbol, + leftadjustment::Symbol, rightadjustment::Symbol) + leftbits = _statisticfloatbits(element, left) + rightbits = _statisticfloatbits(element, right) + comparison === :ieee_total_order && return _comparestatisticnumbers( + _statisticieeekey(leftbits), _statisticieeekey(rightbits)) + (_statisticisnan(leftbits) || _statisticisnan(rightbits)) && return nothing + leftvalue = _adjustedstatisticfloat(leftbits, leftadjustment) + rightvalue = _adjustedstatisticfloat(rightbits, rightadjustment) + return _comparestatisticnumbers(leftvalue, rightvalue) +end + +function _comparestatisticvalues(element::Metadata.SchemaElement, + left::AbstractVector{UInt8}, right::AbstractVector{UInt8}, comparison::Symbol; + left_adjustment::Symbol=:none, right_adjustment::Symbol=:none) + comparison === :unsigned_bytes && return _comparestatisticbytes(left, right) + comparison === :signed && return _comparestatisticnumbers( + _statisticsignedvalue(element, left), _statisticsignedvalue(element, right)) + comparison === :unsigned && return _comparestatisticnumbers( + _statisticunsignedvalue(element, left), _statisticunsignedvalue(element, right)) + comparison === :boolean && return _comparestatisticnumbers(left[1], right[1]) + if comparison === :decimal + element.type_ in (Metadata.Type.INT32, Metadata.Type.INT64) && + return _comparestatisticnumbers(_statisticsignedvalue(element, left), + _statisticsignedvalue(element, right)) + return _comparestatisticdecimalbytes(left, right) + end + comparison in (:floating, :ieee_total_order) && + return _comparestatisticfloats(element, left, right, comparison, + left_adjustment, right_adjustment) + return nothing +end + +function _statisticstokenat(bytes, position::Int, token::String) + tokenbytes = codeunits(token) + position + length(tokenbytes) - 1 <= length(bytes) || return false + for offset in eachindex(tokenbytes) + bytes[position + offset - 1] == tokenbytes[offset] || return false + end + return true +end + +function _statisticsisspace(byte::UInt8) + return byte in (0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20) +end + +function _statisticsskipspace(bytes, position::Int) + while position <= length(bytes) && _statisticsisspace(bytes[position]) + position += 1 + end + return position +end + +function _statisticstrimspaceend(bytes, start::Int, stop::Int) + while stop >= start && _statisticsisspace(bytes[stop]) + stop -= 1 + end + return stop +end + +function _statisticsrangematches(bytes, start::Int, stop::Int, token::String) + stop - start + 1 == ncodeunits(token) || return false + return _statisticstokenat(bytes, start, token) +end + +function _statisticscontainslineterminator(bytes, start::Int, stop::Int) + position = start + while position <= stop + byte = bytes[position] + byte in (0x0a, 0x0d) && return true + byte == 0xc2 && position < stop && bytes[position + 1] == 0x85 && + return true + byte == 0xe2 && position + 2 <= stop && bytes[position + 1] == 0x80 && + bytes[position + 2] in (0xa8, 0xa9) && return true + position += 1 + end + return false +end + +function _statisticsbuildsuffixvalid(bytes, opening::Int, stop::Int, + internalclosing::Bool) + bytes[stop] == 0x29 || return false + internalclosing && return false + position = _statisticsskipspace(bytes, opening + 1) + return position + 4 <= stop && _statisticstokenat(bytes, position, "build") +end + +function _statisticsfindjavamarker(bytes, start::Int, stop::Int) + marker = 0 + versionstart = 0 + nextopening = 0 + nextopeningvalid = false + internalclosing = false + position = stop + while position >= start + byte = bytes[position] + if byte == 0x29 && position < stop + internalclosing = true + position -= 1 + continue + elseif byte == 0x28 + nextopening = position + nextopeningvalid = _statisticsbuildsuffixvalid(bytes, position, + stop, internalclosing) + position -= 1 + continue + elseif !_statisticsisspace(byte) + position -= 1 + continue + end + runstop = position + while position >= start && _statisticsisspace(bytes[position]) + position -= 1 + end + token = runstop + 1 + token + 6 <= stop && _statisticstokenat(bytes, token, "version") || + continue + if iszero(nextopening) || nextopeningvalid + marker = position + 1 + versionstart = token + 7 + end + end + return marker, versionstart +end + +function _statisticsjavasyntaxvalid(bytes, start::Int, stop::Int) + opening = _statisticsfindboundedbyte(bytes, start, stop, 0x28) + opening == 0 && return true + position = _statisticsskipspace(bytes, opening + 1) + position + 4 <= stop && _statisticstokenat(bytes, position, "build") || + return false + position = _statisticsskipspace(bytes, position + 5) + while position <= stop && bytes[position] != 0x29 + position += 1 + end + return position == stop +end + +function _statisticsparsedigits(bytes, position::Int, stop::Int) + position <= stop && 0x30 <= bytes[position] <= 0x39 || + return UInt64(0), position, false + value = UInt64(0) + while position <= stop && 0x30 <= bytes[position] <= 0x39 + digit = UInt64(bytes[position] - 0x30) + value <= div(typemax(UInt64) - digit, UInt64(10)) || + return UInt64(0), position, false + value = value * UInt64(10) + digit + position += 1 + end + return value, position, true +end + +function _statisticsparsetriplet(bytes, position::Int, stop::Int) + major, position, ok = _statisticsparsedigits(bytes, position, stop) + ok && position <= stop && bytes[position] == 0x2e || + return UInt64(0), UInt64(0), UInt64(0), position, false + minor, position, ok = _statisticsparsedigits(bytes, position + 1, stop) + ok && position <= stop && bytes[position] == 0x2e || + return UInt64(0), UInt64(0), UInt64(0), position, false + patch, position, ok = _statisticsparsedigits(bytes, position + 1, stop) + ok || return UInt64(0), UInt64(0), UInt64(0), position, false + return major, minor, patch, position, true +end + +function _statisticsfindboundedbyte(bytes, start::Int, stop::Int, byte::UInt8) + for position in start:stop + bytes[position] == byte && return position + end + return 0 +end + +function _statisticsidentifier(bytes, start::Int, stop::Int) + start <= stop || return false, UInt64(0), true + value = UInt64(0) + maximum = UInt64(typemax(Int32)) + overflowed = false + for position in start:stop + byte = bytes[position] + 0x30 <= byte <= 0x39 || return false, UInt64(0), true + overflowed && continue + digit = UInt64(byte - 0x30) + if value > div(maximum - digit, UInt64(10)) + overflowed = true + continue + end + value = value * UInt64(10) + digit + end + return true, value, !overflowed +end + +function _statisticsprereleasevalid(bytes, start::Int, stop::Int) + position = start + while true + separator = _statisticsfindboundedbyte(bytes, position, stop, 0x2e) + finish = separator == 0 ? stop : separator - 1 + _, _, valid = _statisticsidentifier(bytes, position, finish) + valid || return false + separator == 0 && return true + position = separator + 1 + end +end + +function _statisticsrangecomparison(bytes, start::Int, stop::Int, target::String) + targetbytes = codeunits(target) + position = start + targetposition = firstindex(targetbytes) + while position <= stop && targetposition <= lastindex(targetbytes) + bytes[position] < targetbytes[targetposition] && return Int8(-1) + bytes[position] > targetbytes[targetposition] && return Int8(1) + position += 1 + targetposition += 1 + end + position <= stop && return Int8(1) + targetposition <= lastindex(targetbytes) && return Int8(-1) + return Int8(0) +end + +function _statisticsidentifiercomparison(bytes, start::Int, stop::Int, + target::String, targetnumber::Union{Nothing,UInt64}) + numeric, value, valid = _statisticsidentifier(bytes, start, stop) + valid || throw(AssertionError("validated prerelease identifier overflowed")) + targetnumeric = targetnumber !== nothing + numeric != targetnumeric && return numeric ? Int8(-1) : Int8(1) + numeric && return _comparestatisticnumbers(value, targetnumber::UInt64) + return _statisticsrangecomparison(bytes, start, stop, target) +end + +function _statisticscdhlabelcomparison(bytes, start::Int, stop::Int) + position = start + for targetindex in 1:3 + separator = _statisticsfindboundedbyte(bytes, position, stop, 0x2e) + finish = separator == 0 ? stop : separator - 1 + target = targetindex == 1 ? "cdh5" : targetindex == 2 ? "5" : "0" + targetnumber = targetindex == 1 ? nothing : + targetindex == 2 ? UInt64(5) : UInt64(0) + compared = _statisticsidentifiercomparison(bytes, position, finish, + target, targetnumber) + iszero(compared) || return compared + separator == 0 && return targetindex == 3 ? Int8(0) : Int8(-1) + position = separator + 1 + end + return Int8(1) +end + +function _statisticstrimemptyidentifiers(bytes, start::Int, stop::Int) + while stop >= start && bytes[stop] == 0x2e + stop -= 1 + end + return stop +end + +function _statisticsversionsuffix(bytes, position::Int, stop::Int, + major::UInt64, minor::UInt64, patch::UInt64) + separator = 0 + for index in position:stop + bytes[index] in (0x2b, 0x2d) || continue + separator = index + break + end + plus = _statisticsfindboundedbyte(bytes, position, stop, 0x2b) + plus != 0 && _statisticscontainslineterminator(bytes, plus + 1, stop) && + return false, false, false + unknownstop = separator == 0 ? stop : separator - 1 + unknown = position <= unknownstop + separator == 0 && return true, unknown, false + bytes[separator] == 0x2b && return true, unknown, false + labelstop = plus == 0 ? stop : plus - 1 + labelstart = separator + 1 + _statisticsprereleasevalid(bytes, labelstart, labelstop) || + return false, false, false + comparisonstop = _statisticstrimemptyidentifiers(bytes, labelstart, + labelstop) + cdh = major == 1 && minor == 5 && patch == 0 && !unknown && + _statisticscdhlabelcomparison(bytes, labelstart, comparisonstop) >= 0 + return true, true, cdh +end + +function _statisticsversion(bytes, start::Int, stop::Int) + start <= stop || return _NO_STATISTIC_VERSION + major, minor, patch, position, ok = _statisticsparsetriplet(bytes, start, stop) + ok || return _NO_STATISTIC_VERSION + major <= typemax(Int32) && minor <= typemax(Int32) && patch <= typemax(Int32) || + return _NO_STATISTIC_VERSION + valid, prerelease, cdh = _statisticsversionsuffix(bytes, position, stop, + major, minor, patch) + valid || return _NO_STATISTIC_VERSION + return _StatisticProducerVersion(true, major, minor, patch, prerelease, cdh) +end + +function _statisticsjavaproducer(createdby::Union{Nothing,String}) + createdby === nothing && + return _StatisticProducer(:missing, false, _NO_STATISTIC_VERSION) + bytes = codeunits(createdby) + start = _statisticsskipspace(bytes, 1) + stop = _statisticstrimspaceend(bytes, start, length(bytes)) + start <= stop || + return _StatisticProducer(:missing, false, _NO_STATISTIC_VERSION) + marker, versionstart = _statisticsfindjavamarker(bytes, start, stop) + marker > start && + !_statisticscontainslineterminator(bytes, start, marker - 1) && + _statisticsjavasyntaxvalid(bytes, versionstart, stop) || + return _StatisticProducer(:unknown, false, _NO_STATISTIC_VERSION) + application = if _statisticsrangematches(bytes, start, marker - 1, "parquet-mr") + :parquet_mr + elseif _statisticsrangematches(bytes, start, marker - 1, "parquet-cpp") + :parquet_cpp + else + :other + end + opening = _statisticsfindboundedbyte(bytes, versionstart, stop, 0x28) + versionstop = opening == 0 ? stop : opening - 1 + versionstart = _statisticsskipspace(bytes, versionstart) + versionstop = _statisticstrimspaceend(bytes, versionstart, versionstop) + version = _statisticsversion(bytes, versionstart, versionstop) + return _StatisticProducer(application, true, version) +end + +function _statisticsarrowproducer(createdby::Union{Nothing,String}) + producer = _statisticsjavaproducer(createdby) + producer.parsed && return producer + createdby === nothing && return producer + bytes = codeunits(createdby) + start = _statisticsskipspace(bytes, 1) + stop = _statisticstrimspaceend(bytes, start, length(bytes)) + start <= stop || return producer + application = if _statisticsrangematches(bytes, start, stop, "parquet-mr") + :parquet_mr + elseif _statisticsrangematches(bytes, start, stop, "parquet-cpp") + :parquet_cpp + else + return producer + end + return _StatisticProducer(application, true, _NO_STATISTIC_VERSION) +end + +function _statisticsversionbelow(version::_StatisticProducerVersion, + major::UInt64, minor::UInt64, patch::UInt64) + version.present || return true + current = (version.major, version.minor, version.patch) + cutoff = (major, minor, patch) + current < cutoff && return true + current > cutoff && return false + return version.prerelease +end + +function _statisticscdhexception(version::_StatisticProducerVersion) + version.present || return false + return version.cdh_exception +end + +function _statisticsparquet251affected(producer::_StatisticProducer) + producer.parsed || return true + producer.application === :parquet_mr || return false + _statisticscdhexception(producer.version) && return false + return _statisticsversionbelow(producer.version, UInt64(1), UInt64(8), UInt64(0)) +end + +function _statisticsoldorderaffected(producer::_StatisticProducer) + if producer.application === :parquet_cpp + return _statisticsversionbelow(producer.version, UInt64(1), UInt64(3), UInt64(0)), + :parquet_cpp_pre_1_3 + elseif producer.application === :parquet_mr + return _statisticsversionbelow(producer.version, UInt64(1), UInt64(10), UInt64(0)), + :parquet_mr_pre_1_10 + end + return false, :trusted +end + +function _statisticissignedcomparison(comparison::Symbol) + return comparison in (:signed, :boolean, :decimal, :floating) +end + +function _statisticproducertrust(element::Metadata.SchemaElement, + createdby::Union{Nothing,String}, comparison::Symbol, family::Symbol, lower, upper, + limit::Int64) + family === :none && return _StatisticTrustFact(:trusted, :no_bounds) + binary = element.type_ in (Metadata.Type.BYTE_ARRAY, + Metadata.Type.FIXED_LEN_BYTE_ARRAY) + java = _statisticsjavaproducer(createdby) + if binary && _statisticsparquet251affected(java) + return _StatisticTrustFact(:untrusted, :parquet_251) + end + _statisticissignedcomparison(comparison) && + return _StatisticTrustFact(:trusted, :trusted) + arrow = _statisticsarrowproducer(createdby) + affected, reason = _statisticsoldorderaffected(arrow) + affected || return _StatisticTrustFact(:trusted, :trusted) + lower !== nothing && upper !== nothing && length(lower) <= limit && + length(upper) <= limit && lower == upper && + return _StatisticTrustFact(:trusted, :affected_equal_bounds) + return _StatisticTrustFact(:untrusted, reason) +end + +function _statisticfamily(statistics::Union{Nothing,Metadata.Statistics}) + statistics === nothing && return :none + statistics.min_value !== nothing && return :modern + statistics.max_value !== nothing && return :modern + statistics.min !== nothing && return :deprecated + statistics.max !== nothing && return :deprecated + return :none +end + +function _statisticfamilyvalues(statistics::Union{Nothing,Metadata.Statistics}, + family::Symbol) + family === :none && return nothing, nothing, nothing, nothing + statistics = statistics::Metadata.Statistics + family === :modern && return statistics.min_value, statistics.max_value, + statistics.is_min_value_exact, statistics.is_max_value_exact + return statistics.min, statistics.max, nothing, nothing +end + +function _deprecatedstatisticscompatible(semantics::_StatisticLeafSemantics) + return _statisticissignedcomparison(semantics.comparison) +end + +function _invalidateboundpair(lower::_StatisticBoundFact, + upper::_StatisticBoundFact, reason::Symbol) + return _unknownbound(lower, reason), _unknownbound(upper, reason) +end + +function _widenstatisticzero(bound::_StatisticBoundFact, + adjustment::Symbol) + return _StatisticBoundFact(bound.state, bound.raw, :inexact, :widened_zero, + adjustment) +end + +function _typeorderfloatingbounds(element::Metadata.SchemaElement, + lower::_StatisticBoundFact, upper::_StatisticBoundFact, occupancy::Symbol) + occupancy === :all_nan && (lower.state !== :absent || upper.state !== :absent) && + return _invalidateboundpair(lower, upper, :all_nan_type_order) + if lower.state === :known + bits = _statisticfloatbits(element, lower.raw::Vector{UInt8}) + _statisticisnan(bits) && (lower = _unknownbound(lower, :nan_type_order)) + lower.state === :known && _statisticiszero(bits) && !_statisticisnegative(bits) && + (lower = _widenstatisticzero(lower, :negative_zero)) + end + if upper.state === :known + bits = _statisticfloatbits(element, upper.raw::Vector{UInt8}) + _statisticisnan(bits) && (upper = _unknownbound(upper, :nan_type_order)) + upper.state === :known && _statisticiszero(bits) && _statisticisnegative(bits) && + (upper = _widenstatisticzero(upper, :positive_zero)) + end + return lower, upper +end + +function _ieeefloatingbounds(element::Metadata.SchemaElement, + lower::_StatisticBoundFact, upper::_StatisticBoundFact, occupancy::Symbol) + lowernan = lower.state === :known && + _statisticisnan(_statisticfloatbits(element, lower.raw::Vector{UInt8})) + uppernan = upper.state === :known && + _statisticisnan(_statisticfloatbits(element, upper.raw::Vector{UInt8})) + if occupancy === :all_nan + (!lowernan && lower.state === :known || !uppernan && upper.state === :known) && + return _invalidateboundpair(lower, upper, :ieee_bound_kind) + elseif occupancy === :has_non_nan + (lowernan || uppernan) && + return _invalidateboundpair(lower, upper, :ieee_bound_kind) + else + lowernan && (lower = _unknownbound(lower, :unproven_ieee_nan)) + uppernan && (upper = _unknownbound(upper, :unproven_ieee_nan)) + end + return lower, upper +end + +function _normalizefloatingbounds(element::Metadata.SchemaElement, + lower::_StatisticBoundFact, upper::_StatisticBoundFact, comparison::Symbol, + occupancy::Symbol) + comparison === :floating && + return _typeorderfloatingbounds(element, lower, upper, occupancy) + comparison === :ieee_total_order && + return _ieeefloatingbounds(element, lower, upper, occupancy) + return lower, upper +end + +function _contradictorystatisticbounds(element::Metadata.SchemaElement, + lower::_StatisticBoundFact, upper::_StatisticBoundFact, comparison::Symbol) + lower.state === :known && upper.state === :known || return false + order = _comparestatisticvalues(element, lower.raw::Vector{UInt8}, + upper.raw::Vector{UInt8}, comparison; + left_adjustment=lower.adjustment, right_adjustment=upper.adjustment) + order === nothing && return false + return order > 0 +end + +function _statisticboundgate(family::Symbol, order::_StatisticOrderFact, + semantics::_StatisticLeafSemantics, trust::_StatisticTrustFact, + occupancy::Symbol) + family === :none && return :none + family === :modern && order.declared === :absent && return :missing_order + family === :modern && order.declared === :unknown && return :unknown_order + family === :modern && order.comparison === :undefined && return :undefined_order + family === :deprecated && !_deprecatedstatisticscompatible(semantics) && + return :deprecated_order_mismatch + trust.state === :untrusted && return trust.reason + occupancy === :no_non_null && return :no_non_null + return :valid +end + +function _evaluatestatisticbounds(element::Metadata.SchemaElement, + statistics::Union{Nothing,Metadata.Statistics}, family::Symbol, + semantics::_StatisticLeafSemantics, order::_StatisticOrderFact, + comparison::Symbol, trust::_StatisticTrustFact, occupancy::Symbol, + limits::Limits, budget::_LiveByteBudget) + rawlower, rawupper, lowerflag, upperflag = + _statisticfamilyvalues(statistics, family) + limit = limits.max_statistics_value_bytes + lower = _preparestatisticbound(element, rawlower, family, lowerflag, limit) + upper = _preparestatisticbound(element, rawupper, family, upperflag, limit) + gate = _statisticboundgate(family, order, semantics, trust, occupancy) + gate === :none && return lower, upper + gate === :valid || return _invalidateboundpair(lower, upper, gate) + lower = _validatestatisticbound(lower, semantics, element, limits, budget) + upper = _validatestatisticbound(upper, semantics, element, limits, budget) + lower, upper = _normalizefloatingbounds(element, lower, upper, comparison, occupancy) + _contradictorystatisticbounds(element, lower, upper, comparison) && + return _invalidateboundpair(lower, upper, :contradictory_bounds) + return lower, upper +end + +function _checkstatisticmetadata(node::SchemaNode, metadata::Metadata.ColumnMetaData) + metadata.type_ == node.element.type_ || throw(FormatError( + "statistics column type $(metadata.type_) does not match schema type " * + "$(node.element.type_)")) + metadata.path_in_schema == node.path || throw(FormatError( + "statistics column path $(metadata.path_in_schema) does not match schema path " * + "$(node.path)")) + metadata.num_values >= 0 || throw(FormatError("negative column chunk value count")) + return +end + +function _statisticsfacts(schema::Schema, leafindex::Integer, + createdby::Union{Nothing,String}, + orders::Union{Nothing,AbstractVector{Metadata.ColumnOrder}}, + metadata::Metadata.ColumnMetaData; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + _statisticlimit(limits) + 1 <= leafindex <= length(schema.leaves) || throw(ArgumentError( + "leaf index $leafindex is outside 1:$(length(schema.leaves))")) + index = Int(leafindex) + declared = _validatecolumnorders(schema, index, orders) + node = schema.leaves[index] + _checkstatisticmetadata(node, metadata) + semantics = _leafstatisticsemantics(node.element) + order = _statisticorder(semantics, declared) + statistics = metadata.statistics + nulls, nans, distinct = _validatestatisticcounts(statistics, + metadata.num_values, semantics.floating) + occupancy = _statisticoccupancy(nulls, nans, metadata.num_values, + semantics.floating) + family = _statisticfamily(statistics) + rawlower, rawupper, _, _ = _statisticfamilyvalues(statistics, family) + selectedcomparison = if family === :modern + order.comparison + elseif family === :deprecated && _deprecatedstatisticscompatible(semantics) + semantics.comparison + else + :undefined + end + trust = _statisticproducertrust(node.element, createdby, + selectedcomparison, family, rawlower, rawupper, + limits.max_statistics_value_bytes) + lower, upper = _evaluatestatisticbounds(node.element, statistics, family, + semantics, order, selectedcomparison, trust, occupancy, limits, budget) + return _ColumnStatisticFacts(lower, upper, nulls, nans, distinct, order, + trust, family, selectedcomparison, occupancy) +end diff --git a/src/table.jl b/src/table.jl new file mode 100644 index 0000000..200626f --- /dev/null +++ b/src/table.jl @@ -0,0 +1,477 @@ +import Tables + +mutable struct Table{F<:File,C<:NamedTuple} + file::F + metadata::Metadata.FileMetaData + schema::Schema + columns::C + rows::Int + @atomic closed::Bool +end + +function _tablewritecolumn(name, values::AbstractVector, node::SchemaNode, + limits::Limits) + node.element.type_ === nothing && return _writecolumn(name, values, limits) + length(node.path) == 1 || throw(ArgumentError( + "cannot write a nested primitive as a top-level table column")) + String(name) == node.element.name || throw(ArgumentError( + "table column $(repr(name)) does not match stored schema field " * + repr(node.element.name))) + return _writescalarlogicalcolumn(name, values, node.element, limits) +end + +function _writecolumns(table::Table, limits::Limits, + budget::_LiveByteBudget) + columns = Tables.columns(table) + raw_names = Tables.columnnames(columns) + _reservearray!(budget, Any, length(raw_names)) + names = collect(raw_names) + nodes = table.schema.root.children + length(names) == length(nodes) || throw(ArgumentError( + "table columns no longer match the stored Parquet schema")) + _reservearray!(budget, WriteColumn, length(names)) + output = WriteColumn[] + sizehint!(output, length(names)) + rows = nothing + for (name, node) in zip(names, nodes) + values = Tables.getcolumn(columns, name) + values isa AbstractVector || throw(ArgumentError( + "Parquet columns must be vectors")) + if rows === nothing + rows = length(values) + _checklimit(:container_elements, rows, limits.max_container_elements) + end + length(values) == rows || throw(ArgumentError( + "Parquet columns have different lengths")) + _reservewritenormalization!(budget, values, limits) + push!(output, _tablewritecolumn(name, values, node, limits)) + end + return output, something(rows, 0) +end + +function _writecolumns(table::Table, limits::Limits) + return _writecolumns(table, limits, _LiveByteBudget(limits)) +end + +function _writefields(table::Table, limits::Limits, + budget::_LiveByteBudget) + return _provenancewritefields(table, limits, budget, nothing, false) +end + +function _writefieldsencoded(table::Table, limits::Limits, + budget::_LiveByteBudget, encoding, dictionary::Bool) + return _provenancewritefields(table, limits, budget, encoding, dictionary) +end + +function _writekeyvaluemetadata(table::Table, limits::Limits, + budget::_LiveByteBudget) + metadata = table.metadata.key_value_metadata + metadata === nothing && return nothing + _checklimit(:container_elements, length(metadata), + limits.max_container_elements) + for item in metadata + _checklimit(:string_bytes, ncodeunits(item.key), + limits.max_string_bytes) + item.value === nothing || _checklimit(:string_bytes, + ncodeunits(item.value), limits.max_string_bytes) + end + _reservearray!(budget, Metadata.KeyValue, length(metadata)) + return copy(metadata) +end + +function _readfilemetadata(file::File, limits::Limits, + budget::_LiveByteBudget) + file.footer.encrypted && throw(FormatError("encrypted footers are not supported yet")) + readercharge = _reserveobjects!(budget) + reader = Thrift.Reader(file.footer.bytes; limits=limits, budget=budget) + metadata = try + value = Thrift.decode(reader, Metadata.FileMetaData) + value.encryption_algorithm === nothing || throw(FormatError( + "plaintext-footer encryption is not supported yet")) + Thrift.remaining(reader) == 0 || throw(FormatError( + "file footer has trailing bytes")) + value + catch + charge = _materializedsum(readercharge, + Thrift.materializedcharge(reader)) + _release!(budget, charge) + rethrow() + end + _release!(budget, readercharge) + return metadata +end + +function _readfilemetadata(file::File, limits::Limits) + return _readfilemetadata(file, limits, _LiveByteBudget(limits)) +end + +function _isstringcolumn(node::SchemaNode) + return _logicalkind(node) === :string +end + +function _tablecolumntype(node::SchemaNode) + physical = _physicaleltype(node.element.type_) + return _logicaleltype(node, physical) +end + +function _tablecolumn(node::SchemaNode) + length(node.path) == 1 || throw(FormatError("nested columns are not supported yet")) + T = _tablecolumntype(node) + node.max_repetition_level == 0 || throw(FormatError("repeated columns are not supported yet")) + node.max_definition_level in (0, 1) || + throw(FormatError("nested definition levels are not supported yet")) + optional = node.max_definition_level == 1 + value_type = optional ? Union{Missing,T} : T + if node.element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY && + _logicalkind(node) === nothing + return FixedByteArrayVector(value_type, node.element.type_length) + end + return value_type[] +end + +function _tablelogicalpayloadbytes(values::AbstractVector) + objects = Int64(0) + payload = Int64(0) + for value in values + ismissing(value) && continue + value isa AbstractVector{UInt8} || continue + objects = _materializedsum(objects, _MATERIALIZED_OBJECT_BYTES) + payload = _materializedsum(payload, length(value)) + end + return _materializedsum(objects, payload) +end + +function _tablevalues(node::SchemaNode, values::Vector, limits::Limits, + budget::_LiveByteBudget) + kind = _logicalkind(node) + kind === nothing && return values + physical = _physicaleltype(node.element.type_) + logical = _logicaleltype(node, physical) + outputtype = Missing <: eltype(values) ? Union{Missing,logical} : logical + _reservearray!(budget, outputtype, length(values)) + _reserve!(budget, _tablelogicalpayloadbytes(values)) + kind === :decimal && _reserveobjects!(budget, + count(!ismissing, values)) + return _logicalvalues(node, values; limits=limits) +end + +function _tablevalues(node::SchemaNode, values::Vector, limits::Limits) + return _tablevalues(node, values, limits, _LiveByteBudget(limits)) +end + +function _tablevalues(node::SchemaNode, values::Vector) + return _tablevalues(node, values, Limits()) +end + +function _readtablefield(file::File, metadata::Metadata.FileMetaData, schema::Schema, + rowindex::Int, node::SchemaNode, rows::Int64, limits::Limits, + budget::_LiveByteBudget) + values = readcolumn(file, metadata, schema, rowindex, node.column_index; + limits=limits, budget=budget) + length(values) == rows || throw(FormatError( + "flat column $(node.column_index) has $(length(values)) values for $rows rows")) + return _tablevalues(node, values, limits, budget) +end + +function _tablenames(schema::Schema, limits::Limits, + budget::_LiveByteBudget) + charge = _reservearray!(budget, String, length(schema.root.children)) + try + names = String[] + sizehint!(names, length(schema.root.children)) + for node in schema.root.children + push!(names, node.element.name) + end + return _internschemanames(names, limits, budget) + finally + _release!(budget, charge) + end +end + +function _validatetablerowgroupmetadata(group::Metadata.RowGroup, + rowindex::Int, leafcount::Int, footeroffset::Int64) + group.total_byte_size >= 0 || throw(FormatError( + "row group $rowindex has a negative total byte size")) + compressed = group.total_compressed_size + compressed === nothing || compressed >= 0 || throw(FormatError( + "row group $rowindex has a negative compressed byte size")) + offset = group.file_offset + if offset !== nothing + offset >= 0 || throw(FormatError( + "row group $rowindex has a negative file offset")) + (iszero(offset) || 4 <= offset < footeroffset) || throw(FormatError( + "row group $rowindex file offset $offset is outside the file body")) + end + ordinal = group.ordinal + ordinal === nothing || ordinal >= 0 || throw(FormatError( + "row group $rowindex has a negative ordinal")) + sorting = group.sorting_columns + sorting === nothing && return + for column in sorting + index = Int64(column.column_idx) + 0 <= index < leafcount || throw(FormatError( + "row group $rowindex sorting column index $index is outside " * + "the schema leaf range")) + end + return +end + +function _bloomfilterrange(md::Metadata.ColumnMetaData, + footeroffset::Int64) + offset = md.bloom_filter_offset + length = md.bloom_filter_length + length !== nothing && offset === nothing && throw(FormatError( + "bloom-filter length is present without its offset")) + offset === nothing && return nothing + offset >= 4 || throw(FormatError( + "bloom-filter offset $offset is inside the file header")) + if length === nothing + offset < footeroffset || throw(FormatError( + "bloom-filter offset extends past the footer")) + return (Int64(offset), Int64(1)) + end + length > 0 || throw(FormatError( + "bloom-filter length must be positive, got $length")) + stop = _pageindexrangeend(offset, Int64(length), + "bloom-filter range overflows Int64") + stop <= footeroffset || throw(FormatError( + "bloom-filter range extends past the footer")) + return (Int64(offset), Int64(length)) +end + +function _validatetablechunkranges(metadata::Metadata.FileMetaData, + schema::Schema, footeroffset::Int64) + leafcount = length(schema.leaves) + for (rowindex, group) in enumerate(metadata.row_groups) + _validatetablerowgroupmetadata(group, rowindex, leafcount, + footeroffset) + length(group.columns) == leafcount || throw(FormatError( + "row group $rowindex has $(length(group.columns)) columns for " * + "$leafcount schema leaves")) + for (columnindex, (chunk, leaf)) in enumerate(zip(group.columns, + schema.leaves)) + chunk.file_offset >= 0 || throw(FormatError( + "row group $rowindex column chunk $columnindex has a " * + "negative file offset")) + md = _chunkmetadata(chunk, leaf) + _chunkrange(md, footeroffset) + _bloomfilterrange(md, footeroffset) + end + end + return +end + +function _tablechunkmetrics(metadata::Metadata.FileMetaData, schema::Schema, + leaf::SchemaNode) + entries = Int64(0) + payload = Int64(0) + index = Int(leaf.column_index) + for (rowindex, group) in enumerate(metadata.row_groups) + index <= length(group.columns) || throw(FormatError( + "row group $rowindex has no column chunk for leaf $index")) + md = _chunkmetadata(group.columns[index], leaf) + entries = _materializedsum(entries, md.num_values) + payload = _materializedsum(payload, md.total_uncompressed_size) + end + return entries, payload +end + +function _tablevaluepayload(value) + ismissing(value) && return Int64(0) + value isa AbstractString && return Int64(ncodeunits(value)) + value isa AbstractVector{UInt8} && return Int64(length(value)) + value isa JSONValue && return Int64(length(value.bytes)) + value isa BSONValue && return Int64(length(value.bytes)) + value isa AbstractVector || return Int64(0) + bytes = Int64(0) + for child in value + bytes = _materializedsum(bytes, _tablevaluepayload(child)) + end + return bytes +end + +function _tablefieldpayload(values::AbstractVector) + bytes = Int64(0) + for value in values + bytes = _materializedsum(bytes, _tablevaluepayload(value)) + end + return bytes +end + +function _tablefieldpayloadbaseline(metadata::Metadata.FileMetaData, + schema::Schema, rowindex::Int, node::SchemaNode) + node.element.type_ in (Metadata.Type.BYTE_ARRAY, + Metadata.Type.FIXED_LEN_BYTE_ARRAY) || return Int64(0) + chunk = metadata.row_groups[rowindex].columns[Int(node.column_index)] + md = _chunkmetadata(chunk, node) + return Int64(md.total_uncompressed_size) +end + +function _reserveprimitivecolumn!(budget::_LiveByteBudget, + metadata::Metadata.FileMetaData, schema::Schema, node::SchemaNode, + rows::Int) + T = _tablecolumntype(node) + value_type = node.max_definition_level == 1 ? Union{Missing,T} : T + _reservearray!(budget, value_type, rows) + if node.element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY && + _logicalkind(node) === nothing + _reserveobjects!(budget) + end + if node.element.type_ in (Metadata.Type.BYTE_ARRAY, + Metadata.Type.FIXED_LEN_BYTE_ARRAY) + entries, payload = _tablechunkmetrics(metadata, schema, node) + _reserve!(budget, _materializedproduct(entries, + _MATERIALIZED_OBJECT_BYTES)) + _reserve!(budget, payload) + end + _logicalkind(node) === :decimal && _reserveobjects!(budget, rows) + return +end + +function _tablecolumns(metadata::Metadata.FileMetaData, schema::Schema, + rows::Int, budget::_LiveByteBudget) + nodes = schema.root.children + _reservearray!(budget, Any, length(nodes)) + columns = Any[] + sizehint!(columns, length(nodes)) + for node in nodes + _reserveprimitivecolumn!(budget, metadata, schema, node, rows) + column = _tablecolumn(node) + sizehint!(column, rows) + push!(columns, column) + end + return columns +end + +function _readrowgroups!(columns::Vector, file::File, metadata::Metadata.FileMetaData, + schema::Schema, rows::Int, limits::Limits, budget::_LiveByteBudget) + total = Int64(0) + for (rowindex, group) in enumerate(metadata.row_groups) + group.num_rows >= 0 || throw(FormatError("row group $rowindex has a negative row count")) + group.num_rows <= typemax(Int) || throw(FormatError("row group $rowindex row count overflows")) + length(group.columns) == length(schema.leaves) || + throw(FormatError("row group $rowindex has $(length(group.columns)) columns for $(length(schema.leaves)) schema leaves")) + total = try + Base.checked_add(total, group.num_rows) + catch err + err isa OverflowError || rethrow() + throw(FormatError("row group row count overflows Int64")) + end + total <= rows || throw(FormatError( + "row groups contain more rows than the validated table count")) + _checklimit(:container_elements, total, limits.max_container_elements) + for (fieldindex, node) in enumerate(schema.root.children) + before = _budgetused(budget) + values = _readtablefield(file, metadata, schema, rowindex, node, + group.num_rows, limits, budget) + baseline = _tablefieldpayloadbaseline(metadata, schema, rowindex, + node) + retained = max(Int64(0), _tablefieldpayload(values) - baseline) + append!(columns[fieldindex], values) + transient = _budgetused(budget) - before + retained <= transient || throw(AssertionError( + "final variable-width values exceed their materialization charge")) + transient == retained || _release!(budget, transient - retained) + end + end + total == rows || throw(FormatError( + "row groups contain $total rows but $rows were validated")) + return +end + +function _isflattableplan(plan::_NestedSchemaPlan) + for child in plan.root.children + child isa _NestedLeafPlan || return false + child.source.max_repetition_level == 0 || return false + end + return true +end + +function Table(input; limits::Limits=Limits()) + budget = _LiveByteBudget(limits) + _reserveobjects!(budget, 2) + file = File(input; limits=limits, budget=budget) + try + metadata = _readfilemetadata(file, limits, budget) + metadata.num_rows >= 0 || throw(FormatError("file metadata has a negative row count")) + metadata.num_rows <= typemax(Int) || throw(FormatError("file row count overflows Int")) + _checklimit(:container_elements, metadata.num_rows, limits.max_container_elements) + schema = Schema(metadata; limits=limits, budget=budget) + rows = _validatetablerowgroups(metadata, schema, limits) + _checklimit(:container_elements, rows, limits.max_container_elements) + _validatetablechunkranges(metadata, schema, file.footer.offset) + indexpreflight = _preflightoffsetindexdeclarations(file, metadata, + schema, limits) + indexpreflight = _validatepageindexdeclarationoverlaps!(file, + metadata, schema, indexpreflight, budget) + nested = _nestedplan(schema; limits=limits, budget=budget) + nested.plan_count > 0 || throw(AssertionError( + "nested schema plan has no root")) + _validateoffsetindexes!(file, metadata, schema, limits, budget, + indexpreflight) + names = _tablenames(schema, limits, budget) + if _isflattableplan(nested) + columns = _tablecolumns(metadata, schema, rows, budget) + _readrowgroups!(columns, file, metadata, schema, rows, limits, + budget) + else + root = _readnestedroot(file, metadata, schema, nested, limits, + budget) + columns = root.children + end + length(columns) == length(names) || throw(AssertionError( + "table column count does not match its names")) + _reserveobjects!(budget, 2) + named = NamedTuple{Tuple(names)}(Tuple(columns)) + table = Table(file, metadata, schema, named, rows, false) + finalizer(close!, table) + return table + catch + try + close!(file) + catch + end + rethrow() + end +end + +function close!(table::Table) + (@atomicswap table.closed = true) && return + close!(table.file) + return +end + +function Base.close(table::Table) + close!(table) + return +end + +function Base.length(table::Table) + return table.rows +end + +function Tables.istable(::Type{<:Table}) + return true +end + +function Tables.columnaccess(::Type{<:Table}) + return true +end + +function Tables.columns(table::Table) + return table.columns +end + +function Tables.columnnames(table::Table) + return keys(table.columns) +end + +function Tables.schema(table::Table) + names = Tuple(keys(table.columns)) + types = Tuple(eltype(column) for column in values(table.columns)) + return Tables.Schema(names, types) +end + +function Tables.rowcount(table::Table) + return length(table) +end diff --git a/src/thrift.jl b/src/thrift.jl new file mode 100644 index 0000000..0ff14f0 --- /dev/null +++ b/src/thrift.jl @@ -0,0 +1,894 @@ +module Thrift + +import ..Parquet: Limits, FormatError, LimitError, _LiveByteBudget, + _MATERIALIZED_OBJECT_BYTES, _checklimit, _materializedsum, + _reserve!, _reservearray!, _reserveobjects!, _release! + +# Thrift Compact Protocol type codes (field headers and container element types). +const STOP = 0x00 +const BOOL_TRUE = 0x01 +const BOOL_FALSE = 0x02 +const BYTE = 0x03 +const I16 = 0x04 +const I32 = 0x05 +const I64 = 0x06 +const DOUBLE = 0x07 +const BINARY = 0x08 +const LIST = 0x09 +const SET = 0x0a +const MAP = 0x0b +const STRUCT = 0x0c + +abstract type ThriftEnum end + +""" + RawField + +An undecoded Thrift field preserved for re-emission. `bytes` holds the exact encoded +field header followed by the payload; `headerlength` is the header size (0 when the field +was constructed without a verbatim header) and `previd` is the field id that preceded the +header, which the writer needs to re-emit the verbatim header byte-for-byte. +""" +struct RawField + id::Int16 + type::UInt8 + previd::Int16 + headerlength::Int8 + bytes::Vector{UInt8} +end + +function RawField(id::Integer, type::UInt8, payload::AbstractVector{UInt8}) + return RawField(Int16(id), type, Int16(0), Int8(0), Vector{UInt8}(payload)) +end + +function payload(field::RawField) + return @view field.bytes[(Int(field.headerlength) + 1):end] +end + +function Base.:(==)(a::RawField, b::RawField) + return a.id == b.id && a.type == b.type && payload(a) == payload(b) +end + +function Base.hash(x::RawField, h::UInt) + return hash(payload(x), hash(x.type, hash(x.id, hash(:RawField, h)))) +end + +mutable struct Reader{B<:AbstractVector{UInt8}} + const bytes::B + const start::Int + const last::Int + const limits::Limits + const budget::_LiveByteBudget + pos::Int + depth::Int + headerpos::Int + previd::Int16 + materialized::Int64 +end + +function Reader(bytes::AbstractVector{UInt8}, first::Integer, last::Integer; + limits::Limits=Limits(), budget::_LiveByteBudget=_LiveByteBudget(limits)) + first >= firstindex(bytes) || throw(BoundsError(bytes, first)) + last <= lastindex(bytes) || throw(BoundsError(bytes, last)) + last >= first - 1 || throw(ArgumentError("Thrift reader range is reversed")) + return Reader{typeof(bytes)}(bytes, Int(first), Int(last), limits, budget, + Int(first), 0, Int(first), Int16(0), Int64(0)) +end + +function Reader(bytes::AbstractVector{UInt8}; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) + return Reader(bytes, firstindex(bytes), lastindex(bytes); limits=limits, + budget=budget) +end + +function materializedcharge(r::Reader) + return r.materialized +end + +function _readercharge!(r::Reader, bytes::Integer) + _reserve!(r.budget, bytes) + r.materialized = Base.checked_add(r.materialized, Int64(bytes)) + return Int64(bytes) +end + +function _readerarray!(r::Reader, ::Type{T}, count::Integer; + header::Bool=true) where {T} + bytes = _reservearray!(r.budget, T, count; header=header) + r.materialized = Base.checked_add(r.materialized, bytes) + return bytes +end + +function _readerobjects!(r::Reader, count::Integer=1) + bytes = _reserveobjects!(r.budget, count) + r.materialized = Base.checked_add(r.materialized, bytes) + return bytes +end + +function remaining(r::Reader) + return r.last - r.pos + 1 +end + +function consumed(r::Reader) + return r.pos - r.start +end + +function _need(r::Reader, count::Integer) + count <= remaining(r) && return + throw(FormatError("Thrift data is truncated: $count bytes needed, $(remaining(r)) available")) +end + +function readbyte(r::Reader) + _need(r, 1) + value = @inbounds r.bytes[r.pos] + r.pos += 1 + return value +end + +function _checkvarinttail(byte::UInt8, shift::Int, bits::Int) + (byte & 0x80) == 0x00 || throw(FormatError("Thrift varint is longer than $(bits == 32 ? 5 : 10) bytes")) + (byte & 0x7f) >> (bits - shift) == 0x00 || throw(FormatError("Thrift varint overflows $bits bits")) + return +end + +function readvarint(r::Reader, bits::Int) + maxbytes = bits == 32 ? 5 : 10 + result = UInt64(0) + shift = 0 + for i in 1:maxbytes + byte = readbyte(r) + i == maxbytes && _checkvarinttail(byte, shift, bits) + result |= UInt64(byte & 0x7f) << shift + (byte & 0x80) == 0x00 && return result + shift += 7 + end + throw(FormatError("Thrift varint is longer than $maxbytes bytes")) +end + +function _zigzag32(raw::UInt64) + value = UInt32(raw) + return reinterpret(Int32, (value >> 1) ⊻ (-(value & 0x00000001))) +end + +function _zigzag64(raw::UInt64) + return reinterpret(Int64, (raw >> 1) ⊻ (-(raw & 0x0000000000000001))) +end + +function readi8(r::Reader) + return readbyte(r) % Int8 +end + +function readi16(r::Reader) + value = readi32(r) + typemin(Int16) <= value <= typemax(Int16) || throw(FormatError("Thrift i16 value $value is out of range")) + return Int16(value) +end + +function readi32(r::Reader) + return _zigzag32(readvarint(r, 32)) +end + +function readi64(r::Reader) + return _zigzag64(readvarint(r, 64)) +end + +function readdouble(r::Reader) + _need(r, 8) + value = UInt64(0) + for i in 0:7 + value |= UInt64(@inbounds r.bytes[r.pos + i]) << (8 * i) + end + r.pos += 8 + return reinterpret(Float64, value) +end + +function readbool(r::Reader) + byte = readbyte(r) + byte == BOOL_TRUE && return true + byte == BOOL_FALSE && return false + throw(FormatError("invalid Thrift boolean byte $byte")) +end + +function _readsize(r::Reader, what::String) + raw = readvarint(r, 32) + raw <= typemax(Int32) || throw(FormatError("Thrift $what is negative")) + return Int(raw) +end + +function readbinarylength(r::Reader) + count = _readsize(r, "binary length") + _checklimit(:string_bytes, count, r.limits.max_string_bytes) + _need(r, count) + return count +end + +function readbinary(r::Reader) + count = readbinarylength(r) + _readerarray!(r, UInt8, count) + out = Vector{UInt8}(undef, count) + count == 0 || copyto!(out, 1, r.bytes, r.pos, count) + r.pos += count + return out +end + +function readstring(r::Reader) + count = readbinarylength(r) + _readercharge!(r, _materializedsum(_MATERIALIZED_OBJECT_BYTES, count)) + out = String(view(r.bytes, r.pos:(r.pos + count - 1))) + r.pos += count + return out +end + +function _validtype(type::UInt8) + return BOOL_TRUE <= type <= STRUCT +end + +""" + readfieldheader(r, lastid) -> (id, type) + +Read a field header. Returns `(0, STOP)` at the end of a struct. The header position and +the preceding field id are recorded on the reader so unknown fields can be preserved. +""" +function readfieldheader(r::Reader, lastid::Int16) + r.headerpos = r.pos + r.previd = lastid + byte = readbyte(r) + byte == STOP && return (Int16(0), STOP) + type = byte & 0x0f + delta = byte >> 4 + _validtype(type) || throw(FormatError("invalid Thrift field type code $type")) + delta == 0x00 && return (readi16(r), type) + id = Int(lastid) + Int(delta) + id <= typemax(Int16) || throw(FormatError("Thrift field id overflows Int16")) + return (Int16(id), type) +end + +# Minimum encoded size of one value of a type, used to bound container allocations. +function _minbytes(type::UInt8) + type == DOUBLE && return 8 + return 1 +end + +function _checkcontainer(r::Reader, count::Int, minbytes::Int) + _checklimit(:container_elements, count, r.limits.max_container_elements) + _need(r, Int64(count) * Int64(minbytes)) + return +end + +function readlistheader(r::Reader) + byte = readbyte(r) + type = byte & 0x0f + _validtype(type) || throw(FormatError("invalid Thrift list element type code $type")) + count = Int(byte >> 4) + count == 15 && (count = _readsize(r, "list size")) + _checkcontainer(r, count, _minbytes(type)) + return (count, type) +end + +function readmapheader(r::Reader) + count = _readsize(r, "map size") + count == 0 && return (0, STOP, STOP) + byte = readbyte(r) + keytype = byte >> 4 + valuetype = byte & 0x0f + _validtype(keytype) || throw(FormatError("invalid Thrift map key type code $keytype")) + _validtype(valuetype) || throw(FormatError("invalid Thrift map value type code $valuetype")) + _checkcontainer(r, count, _minbytes(keytype) + _minbytes(valuetype)) + return (count, keytype, valuetype) +end + +function _enterdepth!(r::Reader) + depth = r.depth + 1 + _checklimit(:metadata_depth, depth, r.limits.max_metadata_depth) + r.depth = depth + return +end + +function enter!(r::Reader) + depth = r.depth + 1 + _checklimit(:metadata_depth, depth, r.limits.max_metadata_depth) + _readerobjects!(r) + r.depth = depth + return +end + +function leave!(r::Reader) + r.depth -= 1 + return +end + +function skipfield!(r::Reader, type::UInt8) + (type == BOOL_TRUE || type == BOOL_FALSE) && return + skipvalue!(r, type) + return +end + +function skipvalue!(r::Reader, type::UInt8) + if type == BOOL_TRUE || type == BOOL_FALSE + readbool(r) + elseif type == BYTE + readbyte(r) + elseif type == I16 || type == I32 + readvarint(r, 32) + elseif type == I64 + readvarint(r, 64) + elseif type == DOUBLE + _need(r, 8) + r.pos += 8 + elseif type == BINARY + count = readbinarylength(r) + r.pos += count + elseif type == LIST || type == SET + skiplist!(r) + elseif type == MAP + skipmap!(r) + elseif type == STRUCT + skipstruct!(r) + else + throw(FormatError("invalid Thrift type code $type")) + end + return +end + +function skiplist!(r::Reader) + count, type = readlistheader(r) + _enterdepth!(r) + for _ in 1:count + skipvalue!(r, type) + end + leave!(r) + return +end + +function skipmap!(r::Reader) + count, keytype, valuetype = readmapheader(r) + _enterdepth!(r) + for _ in 1:count + skipvalue!(r, keytype) + skipvalue!(r, valuetype) + end + leave!(r) + return +end + +function skipstruct!(r::Reader) + _enterdepth!(r) + lastid = Int16(0) + while true + id, type = readfieldheader(r, lastid) + type == STOP && break + lastid = id + skipfield!(r, type) + end + leave!(r) + return +end + +function _copyrange(r::Reader, first::Int, last::Int) + count = last - first + 1 + _readerarray!(r, UInt8, count) + out = Vector{UInt8}(undef, count) + count == 0 || copyto!(out, 1, r.bytes, first, count) + return out +end + +""" + readrawfield(r, id, type) + +Skip the payload of the field whose header was just read and return it as a `RawField` +carrying the verbatim header and payload bytes. +""" +function readrawfield(r::Reader, id::Int16, type::UInt8) + headerpos = r.headerpos + previd = r.previd + start = r.pos + skipfield!(r, type) + bytes = _copyrange(r, headerpos, r.pos - 1) + _readerobjects!(r) + _readerarray!(r, RawField, 1) + return RawField(id, type, previd, Int8(start - headerpos), bytes) +end + +function pushunknown!(::Nothing, field::RawField) + return RawField[field] +end + +function pushunknown!(unknown::Vector{RawField}, field::RawField) + push!(unknown, field) + return unknown +end + +function finishunknown(r::Reader, ::Nothing) + _readerarray!(r, RawField, 0) + return RawField[] +end + +function finishunknown(::Reader, unknown::Vector{RawField}) + return unknown +end + +# Preserved fields are stored as `Vector{RawField}`, never `NTuple{N, RawField}`: +# `N` would be attacker-controlled, and tuple `==`/`isequal`/`hash` specialize per +# length, so a footer with thousands of unknown fields would force seconds of +# compilation on the first comparison of decoded metadata. +function Base.convert(::Type{Vector{RawField}}, value::Tuple{Vararg{RawField}}) + return collect(RawField, value) +end + +function missingfield(structname::Symbol, field::Symbol) + throw(FormatError("Thrift struct $structname is missing required field $field")) +end + +function typecode(::Type{Bool}) + return BOOL_TRUE +end + +function typecode(::Type{Int8}) + return BYTE +end + +function typecode(::Type{Int16}) + return I16 +end + +function typecode(::Type{Int32}) + return I32 +end + +function typecode(::Type{Int64}) + return I64 +end + +function typecode(::Type{Float64}) + return DOUBLE +end + +function typecode(::Type{String}) + return BINARY +end + +function typecode(::Type{Vector{UInt8}}) + return BINARY +end + +function typecode(::Type{Vector{T}}) where {T} + return LIST +end + +function typecode(::Type{Vector{Pair{K,V}}}) where {K,V} + return MAP +end + +function matches(type::UInt8, ::Type{Bool}) + return type == BOOL_TRUE || type == BOOL_FALSE +end + +function matches(type::UInt8, ::Type{T}) where {T} + return type == typecode(T) +end + +function readelement(r::Reader, ::Type{Bool}) + return readbool(r) +end + +function readelement(r::Reader, ::Type{Int8}) + return readi8(r) +end + +function readelement(r::Reader, ::Type{Int16}) + return readi16(r) +end + +function readelement(r::Reader, ::Type{Int32}) + return readi32(r) +end + +function readelement(r::Reader, ::Type{Int64}) + return readi64(r) +end + +function readelement(r::Reader, ::Type{Float64}) + return readdouble(r) +end + +function readelement(r::Reader, ::Type{String}) + return readstring(r) +end + +function readelement(r::Reader, ::Type{Vector{UInt8}}) + return readbinary(r) +end + +function readelement(r::Reader, ::Type{Vector{T}}) where {T} + value = readlist(r, T) + value === nothing && throw(FormatError("Thrift nested list element type mismatch")) + return value +end + +function readelement(r::Reader, ::Type{Vector{Pair{K,V}}}) where {K,V} + value = readmap(r, K, V) + value === nothing && throw(FormatError("Thrift nested map element type mismatch")) + return value +end + +""" + readlist(r, T) + +Decode a list or set whose elements decode to `T`. Returns `nothing` without consuming +input when the encoded element type does not match `T`. +""" +function readlist(r::Reader, ::Type{T}) where {T} + start = r.pos + count, type = readlistheader(r) + matches(type, T) || (r.pos = start; return nothing) + _enterdepth!(r) + _readerarray!(r, T, count) + out = Vector{T}(undef, count) + for i in 1:count + out[i] = readelement(r, T) + end + leave!(r) + return out +end + +function readmap(r::Reader, ::Type{K}, ::Type{V}) where {K,V} + start = r.pos + count, keytype, valuetype = readmapheader(r) + if count > 0 && !(matches(keytype, K) && matches(valuetype, V)) + r.pos = start + return nothing + end + _enterdepth!(r) + _readerarray!(r, Pair{K,V}, count) + out = Vector{Pair{K,V}}(undef, count) + for i in 1:count + key = readelement(r, K) + out[i] = key => readelement(r, V) + end + leave!(r) + return out +end + +function decode end + +function decode(bytes::AbstractVector{UInt8}, ::Type{T}; limits::Limits=Limits(), + budget::_LiveByteBudget=_LiveByteBudget(limits)) where {T} + reader = Reader(bytes; limits=limits, budget=budget) + try + return decode(reader, T) + catch + iszero(reader.materialized) || _release!(budget, reader.materialized) + rethrow() + end +end + +struct _WriteCountOverflow <: Exception end + +mutable struct _CountingBuffer + count::Int64 +end + +function _writecount!(buffer::_CountingBuffer, bytes::Integer) + bytes >= 0 || throw(ArgumentError("Thrift byte count must be nonnegative")) + bytes <= typemax(Int64) || throw(_WriteCountOverflow()) + buffer.count = try + Base.checked_add(buffer.count, Int64(bytes)) + catch err + err isa OverflowError || rethrow() + throw(_WriteCountOverflow()) + end + return buffer +end + +function Base.push!(buffer::_CountingBuffer, ::UInt8) + return _writecount!(buffer, 1) +end + +function Base.append!(buffer::_CountingBuffer, + bytes::AbstractVector{UInt8}) + return _writecount!(buffer, length(bytes)) +end + +mutable struct _FixedBuffer + bytes::Vector{UInt8} + position::Int +end + +function _fixedend(buffer::_FixedBuffer, count::Int) + count >= 0 || throw(ArgumentError("Thrift byte count must be nonnegative")) + stop = try + Base.checked_add(buffer.position, count) + catch err + err isa OverflowError || rethrow() + throw(AssertionError( + "Compact Thrift encoding exceeded its counted byte size")) + end + stop <= length(buffer.bytes) || throw(AssertionError( + "Compact Thrift encoding exceeded its counted byte size")) + return stop +end + +function Base.push!(buffer::_FixedBuffer, byte::UInt8) + stop = _fixedend(buffer, 1) + @inbounds buffer.bytes[stop] = byte + buffer.position = stop + return buffer +end + +function Base.append!(buffer::_FixedBuffer, + bytes::AbstractVector{UInt8}) + count = length(bytes) + iszero(count) && return buffer + stop = _fixedend(buffer, count) + source = firstindex(bytes) + destination = buffer.position + 1 + for offset in 0:(count - 1) + @inbounds buffer.bytes[destination + offset] = bytes[source + offset] + end + buffer.position = stop + return buffer +end + +struct Writer{B} + buffer::B +end + +function Writer() + return Writer(UInt8[]) +end + +function _encodedsize(value) + buffer = _CountingBuffer(Int64(0)) + encode!(Writer(buffer), value) + return buffer.count +end + +function _encodefixed!(bytes::Vector{UInt8}, value) + buffer = _FixedBuffer(bytes, 0) + encode!(Writer(buffer), value) + buffer.position == length(bytes) || throw(AssertionError( + "Compact Thrift encoding wrote $(buffer.position) bytes after counting " * + "$(length(bytes))")) + return bytes +end + +function writebyte!(w::Writer, byte::UInt8) + push!(w.buffer, byte) + return +end + +function writevarint!(w::Writer, value::UInt64) + while value >= 0x80 + push!(w.buffer, UInt8(value & 0x7f) | 0x80) + value >>= 7 + end + push!(w.buffer, UInt8(value)) + return +end + +function writei8!(w::Writer, value::Int8) + writebyte!(w, value % UInt8) + return +end + +function writei16!(w::Writer, value::Int16) + writei32!(w, Int32(value)) + return +end + +function writei32!(w::Writer, value::Int32) + writevarint!(w, UInt64(reinterpret(UInt32, (value << 1) ⊻ (value >> 31)))) + return +end + +function writei64!(w::Writer, value::Int64) + writevarint!(w, reinterpret(UInt64, (value << 1) ⊻ (value >> 63))) + return +end + +function writedouble!(w::Writer, value::Float64) + bits = reinterpret(UInt64, value) + for i in 0:7 + push!(w.buffer, UInt8((bits >> (8 * i)) & 0xff)) + end + return +end + +function writebool!(w::Writer, value::Bool) + writebyte!(w, value ? BOOL_TRUE : BOOL_FALSE) + return +end + +function writebinary!(w::Writer, bytes::AbstractVector{UInt8}) + length(bytes) <= typemax(Int32) || throw(ArgumentError("Thrift binary is longer than 2^31 - 1 bytes")) + writevarint!(w, UInt64(length(bytes))) + append!(w.buffer, bytes) + return +end + +function writestring!(w::Writer, value::AbstractString) + writebinary!(w, codeunits(value)) + return +end + +function writefieldheader!(w::Writer, lastid::Int16, id::Int16, type::UInt8) + delta = Int(id) - Int(lastid) + if 0 < delta <= 15 + push!(w.buffer, UInt8(delta << 4) | type) + else + push!(w.buffer, type) + writei16!(w, id) + end + return id +end + +function writestop!(w::Writer) + writebyte!(w, STOP) + return +end + +function writelistheader!(w::Writer, count::Int, type::UInt8) + count <= typemax(Int32) || throw(ArgumentError("Thrift list is longer than 2^31 - 1 elements")) + if count < 15 + push!(w.buffer, UInt8(count << 4) | type) + else + push!(w.buffer, 0xf0 | type) + writevarint!(w, UInt64(count)) + end + return +end + +function writelist!(w::Writer, values::Vector{T}) where {T} + writelistheader!(w, length(values), typecode(T)) + for value in values + writeelement!(w, value) + end + return +end + +function writemap!(w::Writer, pairs::Vector{Pair{K,V}}) where {K,V} + count = length(pairs) + count <= typemax(Int32) || throw(ArgumentError("Thrift map is longer than 2^31 - 1 entries")) + writevarint!(w, UInt64(count)) + count == 0 && return + push!(w.buffer, UInt8(typecode(K) << 4) | typecode(V)) + for (key, value) in pairs + writeelement!(w, key) + writeelement!(w, value) + end + return +end + +function writeelement!(w::Writer, value::Bool) + writebool!(w, value) + return +end + +function writeelement!(w::Writer, value::Int8) + writei8!(w, value) + return +end + +function writeelement!(w::Writer, value::Int16) + writei16!(w, value) + return +end + +function writeelement!(w::Writer, value::Int32) + writei32!(w, value) + return +end + +function writeelement!(w::Writer, value::Int64) + writei64!(w, value) + return +end + +function writeelement!(w::Writer, value::Float64) + writedouble!(w, value) + return +end + +function writeelement!(w::Writer, value::String) + writestring!(w, value) + return +end + +function writeelement!(w::Writer, value::Vector{UInt8}) + writebinary!(w, value) + return +end + +function writeelement!(w::Writer, value::Vector{T}) where {T} + writelist!(w, value) + return +end + +function writeelement!(w::Writer, value::Vector{Pair{K,V}}) where {K,V} + writemap!(w, value) + return +end + +""" + writeraw!(w, lastid, field) -> id + +Re-emit a preserved field. The verbatim header is copied when the preceding field id +matches the one seen at decode time; otherwise a canonical header is synthesized. +""" +function writeraw!(w::Writer, lastid::Int16, field::RawField) + if field.headerlength > 0 && lastid == field.previd + append!(w.buffer, field.bytes) + return field.id + end + writefieldheader!(w, lastid, field.id, field.type) + append!(w.buffer, payload(field)) + return field.id +end + +""" + writeunknownafter!(w, unknown, index, lastid) -> (lastid, index) + +Re-emit, in encounter order, the preserved fields that originally followed field `lastid`. +""" +@inline function writeunknownafter!(w::Writer, unknown::Vector{RawField}, index::Int, + lastid::Int16) + while index <= length(unknown) + field = unknown[index] + (field.headerlength > 0 && field.previd == lastid) || break + lastid = writeraw!(w, lastid, field) + index += 1 + end + return (lastid, index) +end + +@inline function writeunknownrest!(w::Writer, unknown::Vector{RawField}, index::Int, + lastid::Int16) + while index <= length(unknown) + lastid = writeraw!(w, lastid, unknown[index]) + index += 1 + end + return lastid +end + +""" + checkunion(name, known, unknown) + +Reject a decoded Thrift union with more than one member (known or preserved unknown). +""" +function checkunion(structname::Symbol, known::Integer, + unknown::Union{Tuple{Vararg{RawField}}, Vector{RawField}}) + known + length(unknown) <= 1 && return + throw(FormatError("Thrift union $structname has more than one member set")) +end + +function checkunionargs(structname::Symbol, known::Integer, + unknown::Union{Tuple{Vararg{RawField}}, Vector{RawField}}) + known + length(unknown) <= 1 && return + throw(ArgumentError("Thrift union $structname accepts at most one member")) +end + +function encode! end + +function encode(value) + w = Writer() + encode!(w, value) + return w.buffer +end + +function enumnames end + +function name(x::ThriftEnum) + for (value, symbol) in enumnames(typeof(x)) + value == x.value && return symbol + end + return nothing +end + +function Base.show(io::IO, x::ThriftEnum) + symbol = name(x) + modname = nameof(parentmodule(typeof(x))) + symbol === nothing && return print(io, modname, ".T(", x.value, ")") + print(io, modname, ".", symbol) + return +end + +end diff --git a/src/vectors.jl b/src/vectors.jl new file mode 100644 index 0000000..9292d9d --- /dev/null +++ b/src/vectors.jl @@ -0,0 +1,1070 @@ +function _checkfixedvalue(value, width::Int32) + ismissing(value) && return + length(value) == width || + throw(ArgumentError("fixed byte-array value length $(length(value)) differs from width $width")) + return +end + +struct FixedByteArrayVector{T} <: AbstractVector{T} + values::Vector{T} + width::Int32 + + function FixedByteArrayVector{T}(values::Vector{T}, width::Int32) where {T} + T in (Vector{UInt8}, Union{Missing,Vector{UInt8}}) || + throw(ArgumentError("fixed byte-array vector has unsupported element type $T")) + width > 0 || throw(ArgumentError("fixed byte-array width must be positive")) + foreach(value -> _checkfixedvalue(value, width), values) + return new{T}(values, width) + end +end + +function FixedByteArrayVector(::Type{T}, width::Int32) where {T} + return FixedByteArrayVector{T}(T[], width) +end + +function Base.IndexStyle(::Type{<:FixedByteArrayVector}) + return IndexLinear() +end + +function Base.size(column::FixedByteArrayVector) + return size(column.values) +end + +function Base.getindex(column::FixedByteArrayVector, index::Int) + return column.values[index] +end + +function Base.setindex!(column::FixedByteArrayVector, value, index::Int) + _checkfixedvalue(value, column.width) + column.values[index] = value + return value +end + +function Base.push!(column::FixedByteArrayVector, value) + _checkfixedvalue(value, column.width) + push!(column.values, value) + return column +end + +function Base.append!(column::FixedByteArrayVector, values) + foreach(value -> _checkfixedvalue(value, column.width), values) + append!(column.values, values) + return column +end + +function Base.sizehint!(column::FixedByteArrayVector, size::Integer) + sizehint!(column.values, size) + return column +end + +const _NestedIndex = Union{Int32,Int64} + +function _nestedindextype(terminal::Integer) + terminal <= typemax(Int32) && return Int32 + return Int64 +end + +function _nestedindexvalue(value::Integer, label::AbstractString) + value >= 0 || throw(ArgumentError("$label must be nonnegative")) + value <= typemax(Int) || throw(ArgumentError("$label exceeds the Julia index range")) + return Int(value) +end + +function _nestedcheckedint(value, label::AbstractString) + value isa Int || throw(ArgumentError("$label is not an Int")) + value >= 0 || throw(ArgumentError("$label must be nonnegative")) + return value +end + +function _nestedvectorcount(values::AbstractVector, label::AbstractString) + Base.@nospecialize values + sizecount = size(values, 1) + sizecount isa Int || throw(ArgumentError( + "$label size is not an Int")) + sizecount >= 0 || throw(ArgumentError( + "$label size must be nonnegative")) + count = length(values) + count isa Int || throw(ArgumentError("$label length is not an Int")) + count == sizecount || throw(ArgumentError( + "$label length differs from its size")) + return sizecount +end + +function _nestedvectoraxisvalue(value, label::AbstractString) + value isa Int || throw(ArgumentError("$label is not an Int")) + return value +end + +function _nestedvectoraxes(values::AbstractVector, label::AbstractString) + Base.@nospecialize values + first = _nestedvectoraxisvalue(firstindex(values), "$label first index") + last = _nestedvectoraxisvalue(lastindex(values), "$label last index") + return first, last +end + +function _nestedviewcount(first::Int, last::Int, label::AbstractString) + first >= 1 || throw(ArgumentError("$label has an invalid first index")) + emptylast = try + Base.checked_sub(first, 1) + catch error + error isa OverflowError || rethrow() + throw(ArgumentError("$label has an invalid entry span")) + end + last >= emptylast || throw(ArgumentError("$label has an invalid entry span")) + last == emptylast && return 0 + return try + Base.checked_add(Base.checked_sub(last, first), 1) + catch error + error isa OverflowError || rethrow() + throw(ArgumentError("$label entry span exceeds the Julia index range")) + end +end + +function _nestedindices(indices::AbstractVector{<:Integer}, label::AbstractString) + isempty(indices) && throw(ArgumentError("$label must contain an initial zero")) + firstvalue = _nestedindexvalue(first(indices), "$label entry") + iszero(firstvalue) || throw(ArgumentError("$label must start at zero")) + previous = firstvalue + for value in Iterators.drop(indices, 1) + current = _nestedindexvalue(value, "$label entry") + current >= previous || throw(ArgumentError("$label must be nondecreasing")) + previous = current + end + T = _nestedindextype(previous) + output = Vector{T}(undef, length(indices)) + for (index, value) in enumerate(indices) + output[index] = T(value) + end + return output +end + +function _nestedvalidity(validity::Nothing, rows::Int, offsets::Vector{<:_NestedIndex}) + return nothing +end + +function _nestedvalidity(validity::AbstractVector{Bool}, rows::Int, + offsets::Vector{<:_NestedIndex}) + length(validity) == rows || throw(ArgumentError( + "nested validity has $(length(validity)) entries for $rows rows")) + output = BitVector(validity) + for row in 1:rows + output[row] || offsets[row] == offsets[row + 1] || throw(ArgumentError( + "null nested row $row has a nonempty child span")) + end + return output +end + +function _nestedspan(offsets::Vector{<:_NestedIndex}, row::Int) + firstindex = try + Base.checked_add(Int(offsets[row]), 1) + catch error + error isa OverflowError || rethrow() + throw(ArgumentError("nested offset exceeds the Julia index range")) + end + lastindex = Int(offsets[row + 1]) + return firstindex, lastindex +end + +function _validatenestedphysicalshape(values::AbstractVector, terminal::Int, + label::AbstractString) + Base.@nospecialize values + count = _nestedvectorcount(values, label) + count == terminal || throw(ArgumentError( + "$label length differs from its terminal offset")) + first, last = _nestedvectoraxes(values, label) + first == 1 && last == count || throw(ArgumentError( + "$label must use one-based contiguous axes")) + return +end + +function _validatenestedphysicalspan(values::AbstractVector, terminal::Int, + label::AbstractString) + Base.@nospecialize values + _validatenestedphysicalshape(values, terminal, label) + iszero(terminal) && return + checkbounds(Bool, values, 1) && checkbounds(Bool, values, terminal) || + throw(ArgumentError("$label axes do not contain its physical span")) + return +end + +struct ListValue{T} <: AbstractVector{T} + values::AbstractVector{T} + first::Int + last::Int +end + +function _validatelistvalue(value::ListValue) + count = _nestedviewcount(value.first, value.last, "list view") + backing = _nestedvectorcount(value.values, "list view backing vector") + first, last = _nestedvectoraxes(value.values, + "list view backing vector") + first == 1 && last == backing || throw(ArgumentError( + "list view backing vector must use one-based contiguous axes")) + (value.first <= backing || (backing < typemax(Int) && + value.first == backing + 1)) || throw(ArgumentError( + "list view insertion point exceeds its backing vector")) + if !iszero(count) + checkbounds(Bool, value.values, value.first) && + checkbounds(Bool, value.values, value.last) || throw( + ArgumentError( + "list view child span exceeds its backing vector axes")) + end + return +end + +function Base.IndexStyle(::Type{<:ListValue}) + return IndexLinear() +end + +function Base.size(value::ListValue) + return (_nestedviewcount(value.first, value.last, "list view"),) +end + +function Base.getindex(value::ListValue, index::Int) + _validatelistvalue(value) + @boundscheck checkbounds(value, index) + item = value.values[value.first + index - 1] + _validatelistvalue(value) + return item +end + +function _collectnestedview(value::AbstractVector{T}) where {T} + output = Vector{T}(undef, length(value)) + for index in eachindex(value) + output[index] = value[index] + end + return output +end + +function Base.collect(value::ListValue) + return _collectnestedview(value) +end + +function Base.copy(value::ListValue) + return collect(value) +end + +struct ListVector{T,E,O<:_NestedIndex,V<:Union{Nothing,BitVector}} <: + AbstractVector{T} + offsets::Vector{O} + validity::V + values::AbstractVector{E} +end + +function ListVector(offsets::AbstractVector{<:Integer}, values::AbstractVector; + validity::Union{Nothing,AbstractVector{Bool}}=nothing) + Base.@nospecialize values + normalized = _nestedindices(offsets, "list offsets") + rows = length(normalized) - 1 + _validatenestedphysicalshape(values, Int(last(normalized)), "list child") + valid = _nestedvalidity(validity, rows, normalized) + E = eltype(values) + V = ListValue{E} + T = valid === nothing ? V : Union{Missing,V} + return ListVector{T,E,eltype(normalized),typeof(valid)}( + normalized, valid, values) +end + +function ListVector(offsets::AbstractVector{<:Integer}, validity::AbstractVector{Bool}, + values::AbstractVector) + return ListVector(offsets, values; validity=validity) +end + +function Base.IndexStyle(::Type{<:ListVector}) + return IndexLinear() +end + +function Base.size(column::ListVector) + return (length(column.offsets) - 1,) +end + +function _validatelistlocal(column::ListVector) + offsets = column.offsets + isempty(offsets) && throw(ArgumentError( + "list offsets lost their initial zero")) + iszero(first(offsets)) || throw(ArgumentError( + "list offsets no longer start at zero")) + rows = length(offsets) - 1 + column.validity === nothing || length(column.validity) == rows || throw( + ArgumentError("list validity length differs from its row count")) + previous = Int(first(offsets)) + for row in 1:rows + current = Int(offsets[row + 1]) + current >= previous || throw(ArgumentError( + "LIST offsets are no longer nondecreasing")) + column.validity === nothing || column.validity[row] || + current == previous || throw(ArgumentError( + "null LIST row has a nonempty child span")) + previous = current + end + return previous +end + +function _validatelistvector(column::ListVector) + terminal = _validatelistlocal(column) + _validatenestedphysicalspan(column.values, terminal, "LIST child") + _validatelistlocal(column) + return +end + +function Base.getindex(column::ListVector, index::Int) + @boundscheck checkbounds(column, index) + column.validity === nothing || column.validity[index] || return missing + firstindex, lastindex = _nestedspan(column.offsets, index) + return ListValue(column.values, firstindex, lastindex) +end + +struct StructValue + names::Vector{String} + children::Vector{AbstractVector} + index::Int +end + +function _validatestructvaluelocal(value::StructValue, names::Vector{String}, + children::Vector{AbstractVector}, count::Int) + value.names === names && value.children === children || throw( + ArgumentError("struct backing identity changed during access")) + length(children) == count || throw(ArgumentError( + "struct child identity or order changed during access")) + length(names) == count || throw(ArgumentError( + "struct child identity or order changed during access")) + value.index >= 1 || throw(ArgumentError( + "struct view has an invalid row index")) + return +end + +function _validatestructvalue(value::StructValue) + names = value.names + children = value.children + count = length(children) + _validatestructvaluelocal(value, names, children, count) + for (index, child) in enumerate(children) + childcount = _nestedvectorcount(child, "struct view child $index") + first, last = _nestedvectoraxes(child, "struct view child $index") + first == 1 && last == childcount || throw(ArgumentError( + "struct view child $index must use one-based contiguous axes")) + value.index <= childcount || throw(ArgumentError( + "struct view child $index does not contain its row index")) + checkbounds(Bool, child, value.index) || throw(ArgumentError( + "struct view child $index axes do not contain its row index")) + end + _validatestructvaluelocal(value, names, children, count) + return +end + +function Base.length(value::StructValue) + return length(value.children) +end + +function Base.eltype(::Type{<:StructValue}) + return Pair{String,Any} +end + +function _structvaluechild(value::StructValue, index::Int) + names = value.names + children = value.children + count = length(children) + _validatestructvaluelocal(value, names, children, count) + child = children[index] + childcount = _nestedvectorcount(child, "struct view child $index") + first, last = _nestedvectoraxes(child, "struct view child $index") + first == 1 && last == childcount || throw(ArgumentError( + "struct view child $index must use one-based contiguous axes")) + value.index <= childcount || throw(ArgumentError( + "struct view child $index does not contain its row index")) + checkbounds(Bool, child, value.index) || throw(ArgumentError( + "struct view child $index axes do not contain its row index")) + _validatestructvaluelocal(value, names, children, count) + children[index] === child || throw(ArgumentError( + "struct child identity or order changed during access")) + return child +end + +function Base.getindex(value::StructValue, index::Int) + @boundscheck 1 <= index <= length(value) || throw(BoundsError(value, index)) + child = _structvaluechild(value, index) + item = child[value.index] + _structvaluechild(value, index) === child || throw(ArgumentError( + "struct child identity or order changed during access")) + return item +end + +function _structfieldindex(value::StructValue, name::AbstractString) + found = 0 + for index in eachindex(value.names) + value.names[index] == name || continue + iszero(found) || throw(ArgumentError("struct field name $(repr(name)) is ambiguous")) + found = index + end + iszero(found) && throw(KeyError(name)) + return found +end + +function Base.getindex(value::StructValue, name::AbstractString) + return value[_structfieldindex(value, name)] +end + +function Base.getindex(value::StructValue, name::Symbol) + return value[String(name)] +end + +function Base.iterate(value::StructValue, index::Int=1) + index > length(value) && return nothing + return value.names[index] => value[index], index + 1 +end + +function Base.collect(value::StructValue) + output = Vector{Pair{String,Any}}(undef, length(value)) + for index in eachindex(value.names) + output[index] = value.names[index] => value[index] + end + return output +end + +function Base.copy(value::StructValue) + return collect(value) +end + +function Base.:(==)(left::StructValue, right::StructValue) + length(left) == length(right) || return false + result = true + for index in 1:length(left) + left.names[index] == right.names[index] || return false + equal = left[index] == right[index] + equal === false && return false + equal === missing && (result = missing) + end + return result +end + +function Base.isequal(left::StructValue, right::StructValue) + length(left) == length(right) || return false + for index in 1:length(left) + isequal(left.names[index], right.names[index]) || return false + isequal(left[index], right[index]) || return false + end + return true +end + +function Base.hash(value::StructValue, seed::UInt) + output = hash(:ParquetStructValue, seed) + output = hash(length(value), output) + for index in 1:length(value) + output = hash(value.names[index], output) + output = hash(value[index], output) + end + return output +end + +function (::Type{NamedTuple})(value::StructValue) + length(unique(value.names)) == length(value.names) || throw(ArgumentError( + "cannot convert a struct with duplicate field names to NamedTuple")) + names = try + Tuple(Symbol(name) for name in value.names) + catch err + err isa ArgumentError || rethrow() + throw(ArgumentError("cannot convert a struct with an invalid field name to NamedTuple")) + end + values = ntuple(index -> value[index], length(value)) + return NamedTuple{names}(values) +end + +struct StructVector{T,R<:Union{Nothing,Vector{Int32},Vector{Int64}}} <: AbstractVector{T} + names::Vector{String} + ranks::R + children::Vector{AbstractVector} + rows::Int +end + +function _structchildren(children::Union{Tuple,AbstractVector}) + output = Vector{AbstractVector}(undef, length(children)) + for (index, child) in enumerate(children) + child isa AbstractVector || throw(ArgumentError("struct child $index is not a vector")) + output[index] = child + end + return output +end + +function _structrows(children::Vector{AbstractVector}, rows) + if isempty(children) + rows === nothing && throw(ArgumentError( + "a zero-field required struct needs an explicit row count")) + return _nestedindexvalue(rows, "struct row count") + end + expected = _nestedvectorcount(first(children), "struct child 1") + _validatenestedphysicalshape(first(children), expected, "struct child 1") + rows === nothing || _nestedindexvalue(rows, "struct row count") == expected || + throw(ArgumentError("struct row count differs from its child length")) + return expected +end + +function _validatechildren(names::Vector{String}, children::Vector{AbstractVector}, expected::Int) + length(names) == length(children) || throw(ArgumentError( + "struct has $(length(names)) names for $(length(children)) children")) + for (index, child) in enumerate(children) + _validatenestedphysicalshape(child, expected, "struct child $index") + end + return +end + +function _structranks(ranks::AbstractVector{<:Integer}) + normalized = _nestedindices(ranks, "struct ranks") + for index in 1:(length(normalized) - 1) + difference = normalized[index + 1] - normalized[index] + difference <= 1 || throw(ArgumentError( + "struct rank difference at row $index exceeds one")) + end + return normalized +end + +function StructVector(names::AbstractVector{<:AbstractString}, + children::Union{Tuple,AbstractVector}; + ranks::Union{Nothing,AbstractVector{<:Integer}}=nothing, rows=nothing) + normalizednames = String[String(name) for name in names] + normalizedchildren = _structchildren(children) + if ranks === nothing + rowcount = _structrows(normalizedchildren, rows) + _validatechildren(normalizednames, normalizedchildren, rowcount) + return StructVector{StructValue,Nothing}( + normalizednames, nothing, normalizedchildren, rowcount) + end + isempty(normalizedchildren) && throw(ArgumentError( + "an optional zero-field struct has unobservable presence")) + normalizedranks = _structranks(ranks) + rowcount = length(normalizedranks) - 1 + rows === nothing || _nestedindexvalue(rows, "struct row count") == rowcount || + throw(ArgumentError("struct row count differs from its rank length")) + _validatechildren(normalizednames, normalizedchildren, Int(last(normalizedranks))) + T = Union{Missing,StructValue} + return StructVector{T,typeof(normalizedranks)}( + normalizednames, normalizedranks, normalizedchildren, rowcount) +end + +function StructVector(names::AbstractVector{<:AbstractString}, + ranks::AbstractVector{<:Integer}, children::Union{Tuple,AbstractVector}; rows=nothing) + return StructVector(names, children; ranks=ranks, rows=rows) +end + +function Base.IndexStyle(::Type{<:StructVector}) + return IndexLinear() +end + +function Base.size(column::StructVector) + return (column.rows,) +end + +function _validatestructlocal(column::StructVector) + names = column.names + children = column.children + count = length(children) + length(names) == count || throw(ArgumentError( + "struct child identity or order changed: vector has a different " * + "number of names and children")) + column.rows >= 0 || throw(ArgumentError( + "struct vector has a negative row count")) + if column.ranks === nothing + return column.rows + end + isempty(children) && throw(ArgumentError( + "optional zero-field struct has unobservable presence")) + ranks = column.ranks + length(ranks) == column.rows + 1 || throw(ArgumentError( + "struct rank length differs from its row count")) + isempty(ranks) && throw(ArgumentError( + "struct ranks lost their initial zero")) + iszero(first(ranks)) || throw(ArgumentError( + "struct ranks no longer start at zero")) + previous = Int(first(ranks)) + for row in 1:column.rows + current = Int(ranks[row + 1]) + current >= previous || throw(ArgumentError( + "struct ranks are no longer nondecreasing")) + current - previous <= 1 || throw(ArgumentError( + "struct rank difference exceeds one")) + previous = current + end + return previous +end + +function _validatestructvector(column::StructVector) + terminal = _validatestructlocal(column) + label = column.ranks === nothing ? "required" : "optional" + for (index, child) in enumerate(column.children) + _nestedvectorcount(child, "struct child $index") == terminal || + throw(ArgumentError( + "$label struct child $index length differs from its " * + (label == "required" ? "row count" : "terminal rank"))) + _validatenestedphysicalspan(child, terminal, + "$label struct child $index") + end + _validatestructlocal(column) + return +end + +function Base.getindex(column::StructVector, index::Int) + @boundscheck checkbounds(column, index) + if column.ranks === nothing + return StructValue(column.names, column.children, index) + end + firstindex = Int(column.ranks[index]) + lastindex = Int(column.ranks[index + 1]) + firstindex == lastindex && return missing + return StructValue(column.names, column.children, lastindex) +end + +struct MapValue{K,V,HasValues} <: AbstractVector{Pair{K,V}} + keys::AbstractVector{K} + values::Union{Nothing,AbstractVector{V}} + first::Int + last::Int +end + +function _validatemapvaluekeys(value::MapValue, count::Int) + keycount = _nestedvectorcount(value.keys, "map view key vector") + keyfirst, keylast = _nestedvectoraxes(value.keys, "map view key vector") + keyfirst == 1 && keylast == keycount || throw(ArgumentError( + "map view key vector must use one-based contiguous axes")) + (value.first <= keycount || (keycount < typemax(Int) && + value.first == keycount + 1)) || throw(ArgumentError( + "map view insertion point exceeds its key vector")) + if !iszero(count) + checkbounds(Bool, value.keys, value.first) && + checkbounds(Bool, value.keys, value.last) || throw(ArgumentError( + "map view entry span exceeds its key-vector axes")) + end + return keycount +end + +function _validatemapvaluevalues(value::MapValue{K,V,HasValues}, count::Int, + keycount::Int) where {K,V,HasValues} + if HasValues === true + values = value.values + values === nothing && throw(ArgumentError( + "map view lost its value vector")) + valuecount = _nestedvectorcount(values, "map view value vector") + valuefirst, valuelast = _nestedvectoraxes(values, + "map view value vector") + valuefirst == 1 && valuelast == valuecount || throw(ArgumentError( + "map view value vector must use one-based contiguous axes")) + valuecount == keycount || throw(ArgumentError( + "map view key and value lengths differ")) + (value.first <= valuecount || (valuecount < typemax(Int) && + value.first == valuecount + 1)) || throw(ArgumentError( + "map view insertion point exceeds its value vector")) + if !iszero(count) + checkbounds(Bool, values, value.first) && + checkbounds(Bool, values, value.last) || throw(ArgumentError( + "map view entry span exceeds its value-vector axes")) + end + else + value.values === nothing || throw(ArgumentError( + "key-only map view gained a value vector")) + end + return +end + +function _validatemapvalue(value::MapValue{K,V,HasValues}) where {K,V,HasValues} + HasValues isa Bool || throw(ArgumentError( + "map view has a non-Boolean value-vector discriminator")) + Missing <: K && throw(ArgumentError( + "map view key type cannot include Missing")) + HasValues === false && V !== Missing && throw(ArgumentError( + "key-only map view must use Missing as its value type")) + count = _nestedviewcount(value.first, value.last, "map view") + keycount = _validatemapvaluekeys(value, count) + _validatemapvaluevalues(value, count, keycount) + keycount = _validatemapvaluekeys(value, count) + _validatemapvaluevalues(value, count, keycount) + return +end + +function Base.IndexStyle(::Type{<:MapValue}) + return IndexLinear() +end + +function Base.size(value::MapValue) + return (_nestedviewcount(value.first, value.last, "map view"),) +end + +function Base.getindex(value::MapValue{K,V,true}, index::Int) where {K,V} + _validatemapvalue(value) + @boundscheck checkbounds(value, index) + physical = value.first + index - 1 + keys = value.keys + values = something(value.values) + key = keys[physical] + value.keys === keys && value.values === values || throw(ArgumentError( + "map view changed its backing vectors during key access")) + _validatemapvalue(value) + item = values[physical] + _validatemapvalue(value) + return Pair{K,V}(key, item) +end + +function Base.getindex(value::MapValue{K,Missing,false}, index::Int) where {K} + _validatemapvalue(value) + @boundscheck checkbounds(value, index) + physical = value.first + index - 1 + key = value.keys[physical] + _validatemapvalue(value) + return Pair{K,Missing}(key, missing) +end + +function Base.getindex(value::MapValue{K,V,HasValues}, index::Int) where + {K,V,HasValues} + _validatemapvalue(value) + throw(ArgumentError("map view has invalid type parameters")) +end + +function Base.collect(value::MapValue) + return _collectnestedview(value) +end + +function Base.copy(value::MapValue) + return collect(value) +end + +struct MapVector{T,K,V,HasValues,O<:_NestedIndex, + Validity<:Union{Nothing,BitVector}} <: AbstractVector{T} + offsets::Vector{O} + validity::Validity + keys::AbstractVector{K} + values::Union{Nothing,AbstractVector{V}} +end + +function _validatemapvalues(keys::AbstractVector, values::Nothing) + Base.@nospecialize keys + count = _nestedvectorcount(keys, "map key") + _validatenestedphysicalshape(keys, count, "map key") + return count +end + +function _validatemapvalues(keys::AbstractVector, values::AbstractVector) + Base.@nospecialize keys values + count = _nestedvectorcount(keys, "map key") + _validatenestedphysicalshape(keys, count, "map key") + _validatenestedphysicalshape(values, count, "map value") + return count +end + +function MapVector(offsets::AbstractVector{<:Integer}, keys::AbstractVector, + values::Union{Nothing,AbstractVector}=nothing; + validity::Union{Nothing,AbstractVector{Bool}}=nothing) + Base.@nospecialize keys values + Missing <: eltype(keys) && throw(ArgumentError("map key type cannot include Missing")) + count = _validatemapvalues(keys, values) + any(ismissing, keys) && throw(ArgumentError("map keys cannot be missing")) + _validatemapvalues(keys, values) == count || throw(ArgumentError( + "map key count changed during validation")) + normalized = _nestedindices(offsets, "map offsets") + rows = length(normalized) - 1 + Int(last(normalized)) == count || throw(ArgumentError( + "map terminal offset $(last(normalized)) differs from key length $count")) + valid = _nestedvalidity(validity, rows, normalized) + K = eltype(keys) + hasvalues = values !== nothing + V = hasvalues ? eltype(values) : Missing + R = MapValue{K,V,hasvalues} + T = valid === nothing ? R : Union{Missing,R} + return MapVector{T,K,V,hasvalues,eltype(normalized),typeof(valid)}( + normalized, valid, keys, values) +end + +function MapVector(offsets::AbstractVector{<:Integer}, validity::AbstractVector{Bool}, + keys::AbstractVector, values::Union{Nothing,AbstractVector}) + return MapVector(offsets, keys, values; validity=validity) +end + +function Base.IndexStyle(::Type{<:MapVector}) + return IndexLinear() +end + +function Base.size(column::MapVector) + return (length(column.offsets) - 1,) +end + +function _validatemaplocal(column::MapVector{T,K,V,HasValues}) where + {T,K,V,HasValues} + HasValues isa Bool || throw(ArgumentError( + "map vector has a non-Boolean value-vector discriminator")) + Missing <: K && throw(ArgumentError( + "map vector key type cannot include Missing")) + HasValues === false && V !== Missing && throw(ArgumentError( + "key-only map vector must use Missing as its value type")) + offsets = column.offsets + isempty(offsets) && throw(ArgumentError( + "map offsets lost their initial zero")) + iszero(first(offsets)) || throw(ArgumentError( + "map offsets no longer start at zero")) + rows = length(offsets) - 1 + column.validity === nothing || length(column.validity) == rows || throw( + ArgumentError("map validity length differs from its row count")) + previous = Int(first(offsets)) + for row in 1:rows + current = Int(offsets[row + 1]) + current >= previous || throw(ArgumentError( + "MAP offsets are no longer nondecreasing")) + column.validity === nothing || column.validity[row] || + current == previous || throw(ArgumentError( + "null MAP row has a nonempty entry span")) + previous = current + end + return previous +end + +function _validatemapvector(column::MapVector{T,K,V,HasValues}) where + {T,K,V,HasValues} + terminal = _validatemaplocal(column) + _validatenestedphysicalspan(column.keys, terminal, "MAP key") + if HasValues === true + values = column.values + values === nothing && throw(ArgumentError( + "map vector lost its values")) + _validatenestedphysicalspan(values, terminal, "MAP value") + else + column.values === nothing || throw(ArgumentError( + "key-only map vector gained values")) + end + _validatemaplocal(column) + return +end + +function Base.getindex(column::MapVector{T,K,V,HasValues}, index::Int) where + {T,K,V,HasValues} + HasValues isa Bool || throw(ArgumentError( + "map vector has a non-Boolean value-vector discriminator")) + Missing <: K && throw(ArgumentError( + "map vector key type cannot include Missing")) + HasValues === false && V !== Missing && throw(ArgumentError( + "key-only map vector must use Missing as its value type")) + @boundscheck checkbounds(column, index) + column.validity === nothing || column.validity[index] || return missing + firstindex, lastindex = _nestedspan(column.offsets, index) + return MapValue{K,V,HasValues}( + column.keys, column.values, firstindex, lastindex) +end + +function maplookup(value::MapValue, key) + for index in length(value):-1:1 + pair = value[index] + isequal(pair.first, key) && return pair.second + end + throw(KeyError(key)) +end + +function maplookup(value::MapValue, key, default) + for index in length(value):-1:1 + pair = value[index] + isequal(pair.first, key) && return pair.second + end + return default +end + +struct _StableByteKey <: AbstractVector{UInt8} + bytes::String +end + +function Base.IndexStyle(::Type{_StableByteKey}) + return IndexLinear() +end + +function Base.size(value::_StableByteKey) + return (ncodeunits(value.bytes),) +end + +function Base.getindex(value::_StableByteKey, index::Int) + @boundscheck checkbounds(value, index) + return codeunit(value.bytes, index) +end + +struct _StableListKey <: AbstractVector{Any} + values::Core.SimpleVector +end + +function Base.IndexStyle(::Type{_StableListKey}) + return IndexLinear() +end + +function Base.size(value::_StableListKey) + return (length(value.values),) +end + +function Base.getindex(value::_StableListKey, index::Int) + @boundscheck checkbounds(value, index) + return value.values[index] +end + +struct _StableStructKey + names::Core.SimpleVector + values::Core.SimpleVector +end + +function _structkeylength(value::StructValue) + return length(value) +end + +function _structkeylength(value::_StableStructKey) + return length(value.names) +end + +function _structkeyname(value::StructValue, index::Int) + return value.names[index] +end + +function _structkeyname(value::_StableStructKey, index::Int) + return value.names[index] +end + +function _structkeyvalue(value::StructValue, index::Int) + return value[index] +end + +function _structkeyvalue(value::_StableStructKey, index::Int) + return value.values[index] +end + +function _structkeyisequal(left, right) + length = _structkeylength(left) + length == _structkeylength(right) || return false + for index in 1:length + isequal(_structkeyname(left, index), _structkeyname(right, index)) || return false + isequal(_structkeyvalue(left, index), _structkeyvalue(right, index)) || return false + end + return true +end + +function Base.isequal(left::_StableStructKey, right::_StableStructKey) + return _structkeyisequal(left, right) +end + +function Base.isequal(left::_StableStructKey, right::StructValue) + return _structkeyisequal(left, right) +end + +function Base.isequal(left::StructValue, right::_StableStructKey) + return _structkeyisequal(left, right) +end + +function Base.hash(value::_StableStructKey, seed::UInt) + output = hash(:ParquetStructValue, seed) + output = hash(_structkeylength(value), output) + for index in 1:_structkeylength(value) + output = hash(_structkeyname(value, index), output) + output = hash(_structkeyvalue(value, index), output) + end + return output +end + +function _stablekeyerror(value) + throw(ArgumentError( + "map key type $(typeof(value)) does not have stable content hash semantics")) +end + +function _stablekeysimplevector(value) + output = [] + sizehint!(output, length(value)) + for item in value + push!(output, _snapshotmapkey(item)) + end + return Core.svec(output...) +end + +function _snapshotmapkey(value::ListValue) + return _StableListKey(_stablekeysimplevector(value)) +end + +function _snapshotmapkey(value::StructValue) + names = Core.svec(value.names...) + output = [] + sizehint!(output, length(value)) + for index in 1:length(value) + push!(output, _snapshotmapkey(value[index])) + end + return _StableStructKey(names, Core.svec(output...)) +end + +function _snapshotmapkey(value::MapValue) + return _stablekeyerror(value) +end + +function _snapshotmapkey(value::Tuple) + return map(_snapshotmapkey, value) +end + +function _snapshotmapkey(value::NamedTuple) + mapped = map(_snapshotmapkey, values(value)) + return NamedTuple{keys(value)}(mapped) +end + +function _snapshotmapkey(value::AbstractString) + return String(value) +end + +function _snapshotmapkey(value::Symbol) + return value +end + +function _snapshotmapkey(value::AbstractVector{UInt8}) + return _StableByteKey(String(collect(value))) +end + +function _snapshotmapkey(value::AbstractArray) + return _stablekeyerror(value) +end + +function _snapshotmapkey(value::Missing) + return missing +end + +function _snapshotmapkey(value::Decimal) + return copy(value) +end + +function _snapshotmapkey(value::JSONValue) + return copy(value) +end + +function _snapshotmapkey(value::BSONValue) + return copy(value) +end + +function _snapshotmapkey(value::_StableByteKey) + return value +end + +function _snapshotmapkey(value::_StableListKey) + return value +end + +function _snapshotmapkey(value::_StableStructKey) + return value +end + +function _snapshotmapkey(value) + value === nothing && return _stablekeyerror(value) + type = typeof(value) + Base.ismutabletype(type) && return _stablekeyerror(value) + (isbitstype(type) || fieldcount(type) == 0) && return value + return _stablekeyerror(value) +end + +function (::Type{Dict})(value::MapValue{K,V}) where {K,V} + keys = [] + values = Vector{V}(undef, length(value)) + sizehint!(keys, length(value)) + Key = Union{} + for (index, pair) in enumerate(value) + ismissing(pair.first) && throw(ArgumentError("map keys cannot be missing")) + key = _snapshotmapkey(pair.first) + push!(keys, key) + values[index] = pair.second + Key = typejoin(Key, typeof(key)) + end + isempty(keys) && (Key = Any) + output = Dict{Key,V}() + sizehint!(output, length(value)) + for index in eachindex(keys, values) + output[keys[index]] = values[index] + end + return output +end diff --git a/src/write.jl b/src/write.jl new file mode 100644 index 0000000..00606ab --- /dev/null +++ b/src/write.jl @@ -0,0 +1,2026 @@ +import Tables + +struct WriteColumn{V} + name::String + values::V + physical::Metadata.Type.T + type_length::Union{Nothing,Int32} + optional::Bool + logical::Union{Nothing, Metadata.LogicalType} + converted::Union{Nothing, Metadata.ConvertedType.T} + path::Vector{String} + repetitions::Union{Nothing,AbstractVector{UInt64}} + definitions::Union{Nothing,AbstractVector{UInt64}} + max_repetition_level::Int16 + max_definition_level::Int16 + rows::Int + schema::Vector{Metadata.SchemaElement} +end + +struct WriteFieldPlan + schema::Vector{Metadata.SchemaElement} + leaves::Vector{WriteColumn} +end + +struct WriteLeafPlan + ordinal::Int32 + path::Vector{String} + column::WriteColumn + entry_offsets::Vector{Int64} + dense_offsets::Vector{Int64} + payload_offsets::Vector{Int64} +end + +function WriteLeafPlan(ordinal::Int32, path::Vector{String}, column::WriteColumn) + return WriteLeafPlan(ordinal, path, column, Int64[], Int64[], Int64[]) +end + +struct WriteRowGroupPlan + rows::Int + leaves::Vector{WriteLeafPlan} +end + +struct WritePlan + elements::Vector{Metadata.SchemaElement} + schema::Schema + rows::Int + rowgroups::Vector{WriteRowGroupPlan} +end + +struct ColumnPages + bytes::Vector{UInt8} + uncompressed_size::Int64 + data_offset::Int64 + dictionary_offset::Union{Nothing,Int64} + encodings::Vector{Metadata.Encoding.T} + encoding_stats::Vector{Metadata.PageEncodingStats} + page_locations::Vector{Metadata.PageLocation} +end + +function ColumnPages(bytes::Vector{UInt8}, uncompressed_size::Int64, + data_offset::Int64, dictionary_offset::Union{Nothing,Int64}, + encodings::Vector{Metadata.Encoding.T}, + encoding_stats::Vector{Metadata.PageEncodingStats}) + return ColumnPages(bytes, uncompressed_size, data_offset, + dictionary_offset, encodings, encoding_stats, Metadata.PageLocation[]) +end + +struct WriteEncodingChoice + encoding::Union{Nothing,Metadata.Encoding.T} + dictionary::Bool +end + +function _writeleafschema(name::String, physical::Metadata.Type.T, + type_length::Union{Nothing,Int32}, optional::Bool, + logical::Union{Nothing,Metadata.LogicalType}, + converted::Union{Nothing,Metadata.ConvertedType.T}) + repetition = optional ? Metadata.FieldRepetitionType.OPTIONAL : + Metadata.FieldRepetitionType.REQUIRED + return Metadata.SchemaElement( + type_=physical, + type_length=type_length, + repetition_type=repetition, + name=name, + converted_type=converted, + logicalType=logical, + ) +end + +function WriteColumn(name::String, values, physical::Metadata.Type.T, + type_length::Union{Nothing,Int32}, optional::Bool, + logical::Union{Nothing,Metadata.LogicalType}, + converted::Union{Nothing,Metadata.ConvertedType.T}) + definition = Int16(optional ? 1 : 0) + schema = Metadata.SchemaElement[ + _writeleafschema(name, physical, type_length, optional, logical, converted), + ] + return WriteColumn(name, values, physical, type_length, optional, logical, converted, + String[name], nothing, nothing, Int16(0), definition, length(values), schema) +end + +function _writetype(::Type{Bool}) + return Metadata.Type.BOOLEAN +end + +function _writetype(::Type{Int32}) + return Metadata.Type.INT32 +end + +function _writetype(::Type{Int64}) + return Metadata.Type.INT64 +end + +function _writetype(::Type{Float32}) + return Metadata.Type.FLOAT +end + +function _writetype(::Type{Float64}) + return Metadata.Type.DOUBLE +end + +function _writetype(::Type{T}) where {T<:AbstractString} + return Metadata.Type.BYTE_ARRAY +end + +function _writetype(::Type{T}) where {T<:AbstractVector{UInt8}} + return Metadata.Type.BYTE_ARRAY +end + +function _writetype(::Type{NTuple{N,UInt8}}) where {N} + return Metadata.Type.FIXED_LEN_BYTE_ARRAY +end + +function _writetype(::Type{T}) where {T} + throw(ArgumentError("unsupported PLAIN writer element type $T")) +end + +function _writetypelength(::Type{T}) where {T} + return nothing +end + +function _writetypelength(::Type{NTuple{N,UInt8}}) where {N} + N > 0 || throw(ArgumentError("fixed byte-array width must be positive")) + N <= typemax(Int32) || throw(ArgumentError("fixed byte-array width exceeds Int32")) + return Int32(N) +end + +function _stringlogical(::Type{T}) where {T} + T <: AbstractString || return nothing, nothing + logical = Metadata.LogicalType(STRING=Metadata.StringType()) + return logical, Metadata.ConvertedType.UTF8 +end + +function _addlistentries(total::Int, count::Int, limits::Limits) + requested = try + Base.checked_add(total, count) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, requested, limits.max_container_elements) + return requested +end + +function _listdateentrycount(values::AbstractVector, limits::Limits) + entries = 0 + for row in values + if ismissing(row) + entries = _addlistentries(entries, 1, limits) + continue + end + row isa AbstractVector || throw(ArgumentError( + "Parquet LIST rows must be vectors or missing")) + entries = _addlistentries(entries, max(1, length(row)), limits) + for value in row + ismissing(value) || value isa Dates.Date || throw(ArgumentError( + "Parquet DATE list elements must be Date values or missing")) + end + end + return entries +end + +function _listdatecolumn(name, values::AbstractVector, container_type::Type, + limits::Limits) + element_type = Base.nonmissingtype(eltype(container_type)) + element_type == Dates.Date || throw(ArgumentError( + "unsupported nested Parquet writer element type $element_type")) + entries = _listdateentrycount(values, limits) + repetition = UInt64[] + definition = UInt64[] + dense = Int32[] + sizehint!(repetition, entries) + sizehint!(definition, entries) + for row in values + if ismissing(row) + push!(repetition, 0) + push!(definition, 0) + continue + end + row isa AbstractVector || throw(ArgumentError("Parquet LIST rows must be vectors or missing")) + if isempty(row) + push!(repetition, 0) + push!(definition, 1) + continue + end + for (index, value) in enumerate(row) + push!(repetition, isone(index) ? 0 : 1) + if ismissing(value) + push!(definition, 2) + elseif value isa Dates.Date + push!(definition, 3) + push!(dense, _toparquetdate(value)) + else + throw(ArgumentError("Parquet DATE list elements must be Date values or missing")) + end + end + end + length(repetition) == entries || throw(ArgumentError( + "Parquet LIST input changed while it was encoded")) + logical = Metadata.LogicalType(DATE=Metadata.DateType()) + converted = Metadata.ConvertedType.DATE + outer = Metadata.SchemaElement( + repetition_type=Metadata.FieldRepetitionType.OPTIONAL, + name=String(name), + num_children=Int32(1), + converted_type=Metadata.ConvertedType.LIST, + logicalType=Metadata.LogicalType(LIST=Metadata.ListType()), + ) + repeated = Metadata.SchemaElement( + repetition_type=Metadata.FieldRepetitionType.REPEATED, + name="list", + num_children=Int32(1), + ) + element = _writeleafschema("element", Metadata.Type.INT32, nothing, true, + logical, converted) + path = String[String(name), "list", "element"] + schema = Metadata.SchemaElement[outer, repeated, element] + return WriteColumn(String(name), dense, Metadata.Type.INT32, nothing, true, + logical, converted, path, repetition, definition, Int16(1), Int16(3), + length(values), schema) +end + +function _datecolumn(name, values::AbstractVector, limits::Limits) + logical = Metadata.LogicalType(DATE=Metadata.DateType()) + converted = Metadata.ConvertedType.DATE + optional = Missing <: eltype(values) + element = _writeleafschema(String(name), Metadata.Type.INT32, nothing, optional, + logical, converted) + physical = _physicalvalues(element, values; limits=limits) + return WriteColumn(String(name), physical, Metadata.Type.INT32, nothing, optional, + logical, converted) +end + +function _writecolumn(name, values::AbstractVector, limits::Limits) + type = eltype(values) + optional = Missing <: type + type == Missing && return _unknowncolumn(name, values, limits) + value_type = Base.nonmissingtype(type) + if value_type <: AbstractVector && !(value_type <: AbstractVector{UInt8}) + return _listdatecolumn(name, values, value_type, limits) + end + value_type == Dates.Date && return _datecolumn(name, values, limits) + logical = _logicalwritecolumn(name, values, value_type, limits) + logical === nothing || return logical + physical = _writetype(value_type) + type_length = _writetypelength(value_type) + logical, converted = _stringlogical(value_type) + return WriteColumn(String(name), values, physical, type_length, optional, logical, converted) +end + +function _writecolumn(name, values::FixedByteArrayVector, ::Limits) + optional = Missing <: eltype(values) + return WriteColumn(String(name), values, Metadata.Type.FIXED_LEN_BYTE_ARRAY, + values.width, optional, nothing, nothing) +end + +function _writecolumn(name, values::AbstractVector) + return _writecolumn(name, values, Limits()) +end + +function _writecolumn(name, values::FixedByteArrayVector) + return _writecolumn(name, values, Limits()) +end + +function _writervaluepayload(value) + value isa AbstractString && return Int64(ncodeunits(value)) + value isa AbstractVector{UInt8} && return Int64(length(value)) + value isa JSONValue && return Int64(length(value.bytes)) + value isa BSONValue && return Int64(length(value.bytes)) + value isa UUIDs.UUID && return Int64(16) + value isa Interval && return Int64(12) + return Int64(0) +end + +function _reservewritenormalization!(budget::_LiveByteBudget, + values::AbstractVector, limits::Limits) + count = length(values) + _reserveobjects!(budget, 4) + _reservearray!(budget, UInt64, count) + value_type = Base.nonmissingtype(eltype(values)) + if value_type <: AbstractVector && !(value_type <: AbstractVector{UInt8}) + entries = _listdateentrycount(values, limits) + _reservearray!(budget, UInt64, entries) + _reservearray!(budget, UInt64, entries) + _reservearray!(budget, Int32, entries) + return + end + objects = Int64(0) + payload = Int64(0) + for value in values + ismissing(value) && continue + bytes = _writervaluepayload(value) + iszero(bytes) && continue + objects = _materializedsum(objects, _MATERIALIZED_OBJECT_BYTES) + payload = _materializedsum(payload, bytes) + end + _reserve!(budget, _materializedsum(objects, payload)) + if values isa LogicalColumn && values.spec isa _DecimalLogicalColumnSpec && + values.spec.precision > 18 + width = _decimalwritewidth(values.spec.precision, limits) + _reserve!(budget, _materializedproduct(count, + _materializedsum(_MATERIALIZED_ARRAY_HEADER_BYTES, width))) + end + return +end + +function _writecolumnnamebytes(name::Symbol) + return Int64(sizeof(name)) +end + +function _writecolumnnamebytes(name::AbstractString) + return Int64(ncodeunits(name)) +end + +function _writecolumnnamebytes(name) + throw(ArgumentError( + "Parquet column names must be Symbols or strings, got $(typeof(name))")) +end + +function _validatewritecolumnnames(names, budget::_LiveByteBudget) + temporary = Int64(0) + retained = Int64(0) + try + arraycharge = _reservearray!(budget, String, length(names)) + temporary = _materializedsum(temporary, arraycharge) + temporary = _materializedsum(temporary, _reserveobjects!(budget)) + normalized = String[] + sizehint!(normalized, length(names)) + seen = Set{String}() + for raw in names + bytes = _writecolumnnamebytes(raw) + payloadcharge = _materializedproduct(bytes, 2) + objectcharge = _materializedproduct(2, _MATERIALIZED_OBJECT_BYTES) + itemcharge = _materializedsum(objectcharge, payloadcharge) + _reserve!(budget, itemcharge) + temporary = _materializedsum(temporary, itemcharge) + name = String(raw) + name in seen && throw(ArgumentError( + "Parquet column names must be unique")) + push!(normalized, name) + push!(seen, name) + retained = _materializedsum(retained, + _materializedsum(_MATERIALIZED_OBJECT_BYTES, bytes)) + end + _release!(budget, temporary - arraycharge - retained) + return normalized, arraycharge + catch + iszero(temporary) || _release!(budget, temporary) + rethrow() + end +end + +function _writecolumns(table, limits::Limits, budget::_LiveByteBudget) + columns = Tables.columns(table) + raw_names = Tables.columnnames(columns) + rawcharge = _reservearray!(budget, Any, length(raw_names)) + try + names = collect(raw_names) + isempty(names) && throw(ArgumentError( + "a Parquet table must have at least one column")) + normalized, namearraycharge = _validatewritecolumnnames(names, budget) + try + _reservearray!(budget, WriteColumn, length(names)) + output = WriteColumn[] + sizehint!(output, length(names)) + rows = nothing + for (raw, name) in zip(names, normalized) + values = Tables.getcolumn(columns, raw) + values isa AbstractVector || throw(ArgumentError( + "Parquet columns must be vectors")) + if rows === nothing + rows = length(values) + _checklimit(:container_elements, rows, + limits.max_container_elements) + end + length(values) == rows || throw(ArgumentError( + "Parquet columns have different lengths")) + _reservewritenormalization!(budget, values, limits) + push!(output, _writecolumn(name, values, limits)) + end + return output, something(rows, 0) + finally + _release!(budget, namearraycharge) + end + finally + _release!(budget, rawcharge) + end +end + +function _writecolumns(table, limits::Limits) + return _writecolumns(table, limits, _LiveByteBudget(limits)) +end + +function _preflightwriterowcolumncount(count::Int, limits::Limits, + budget::_LiveByteBudget) + _checklimit(:container_elements, count, + limits.max_container_elements) + minimum = _materializedsum( + _materializedarraybytes(String, count), + _materializedarraybytes(Pair{String,AbstractVector}, count)) + minimum = _materializedsum(minimum, + _materializedarraybytes(AbstractVector, count)) + _reserve!(budget, minimum) + _release!(budget, minimum) + return +end + +Base.@noinline function _writerownamedtupleschema(T::Type) + Base.@nospecialize T + return fieldnames(T), fieldtypes(T) +end + +Base.@noinline function _preflightwriteroweltype(rows, limits::Limits, + budget::_LiveByteBudget) + Base.@nospecialize rows + Base.IteratorEltype(typeof(rows)) isa Base.HasEltype || return nothing + T = eltype(rows) + isconcretetype(T) || return nothing + if T <: NamedTuple + _preflightwriterowcolumncount(fieldcount(T), limits, budget) + return T + end + return nothing +end + +Base.@noinline function _writerowschema(rows, limits::Limits, + budget::_LiveByteBudget) + Base.@nospecialize rows + declared = _preflightwriteroweltype(rows, limits, budget) + schema = Tables.schema(rows) + if schema === nothing + declared === nothing && throw(ArgumentError( + "row-oriented Tables sources must declare a schema or a concrete NamedTuple element type")) + return _writerownamedtupleschema(declared) + end + names = schema.names + types = schema.types + names === nothing && throw(ArgumentError( + "row-oriented Tables sources must declare column names")) + types === nothing && throw(ArgumentError( + "row-oriented Tables sources must declare column types")) + _preflightwriterowcolumncount(length(names), limits, budget) + isempty(names) && throw(ArgumentError( + "a Parquet table must have at least one column")) + length(names) == length(types) || throw(ArgumentError( + "row-oriented Tables schema has different name and type counts")) + return names, types +end + +function _writerowcount(rows, limits::Limits) + Base.haslength(typeof(rows)) || return nothing + count = length(rows) + _checklimit(:container_elements, count, + limits.max_container_elements) + return count +end + +function _writerowarraycharge(types, count::Int) + bytes = Int64(0) + for T in types + T isa Type || throw(ArgumentError( + "row-oriented Tables schemas must contain Julia types")) + bytes = _materializedsum(bytes, + _materializedarraybytes(T, count)) + end + return bytes +end + +function _preflightwriterowtypes(names::Vector{String}, types, + limits::Limits, budget::_LiveByteBudget) + start = _budgetused(budget) + try + for (name, T) in zip(names, types) + T isa Type || throw(ArgumentError( + "row-oriented Tables schemas must contain Julia types")) + _nestedwriteshape(name, T, nothing, limits, budget) + end + return + finally + used = _budgetused(budget) + used > start && _release!(budget, used - start) + end +end + +function _writerowarrays(types, count::Int) + columns = AbstractVector[] + sizehint!(columns, length(types)) + for T in types + push!(columns, Vector{T}(undef, count)) + end + return columns +end + +function _writerowretain!(budget::_LiveByteBudget, value, limits::Limits, + depth::Int=1) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + (ismissing(value) || isbits(value)) && return Int64(0) + if value isa Union{JSONValue,BSONValue} + bytes = Int64(length(value.bytes)) + _checklimit(:string_bytes, bytes, limits.max_string_bytes) + charge = _materializedsum( + _materializedproduct(3, _MATERIALIZED_OBJECT_BYTES), bytes) + _reserve!(budget, charge) + return charge + elseif value isa Decimal + bytes = Int64(_twoscomplementwidth(value.unscaled)) + _checklimit(:decimal_bytes, bytes, limits.max_decimal_bytes) + charge = _materializedsum( + _materializedproduct(2, _MATERIALIZED_OBJECT_BYTES), bytes) + _reserve!(budget, charge) + return charge + elseif value isa AbstractString + bytes = Int64(ncodeunits(value)) + _checklimit(:string_bytes, bytes, limits.max_string_bytes) + charge = _materializedsum(_MATERIALIZED_OBJECT_BYTES, bytes) + _reserve!(budget, charge) + return charge + elseif value isa AbstractVector + count = length(value) + _checklimit(:container_elements, count, + limits.max_container_elements) + charge = _materializedarraybytes(eltype(value), count) + _reserve!(budget, charge) + for item in value + charge = _materializedsum(charge, + _writerowretain!(budget, item, limits, depth + 1)) + end + return charge + elseif value isa NamedTuple + charge = _reserveobjects!(budget) + for item in values(value) + charge = _materializedsum(charge, + _writerowretain!(budget, item, limits, depth + 1)) + end + return charge + elseif value isa Pair + charge = _reserveobjects!(budget) + charge = _materializedsum(charge, + _writerowretain!(budget, first(value), limits, depth + 1)) + return _materializedsum(charge, + _writerowretain!(budget, last(value), limits, depth + 1)) + elseif value isa AbstractDict + count = length(value) + _checklimit(:container_elements, count, + limits.max_container_elements) + charge = _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _materializedproduct(count, 4 * _MATERIALIZED_OBJECT_BYTES)) + _reserve!(budget, charge) + for (key, item) in value + charge = _materializedsum(charge, + _writerowretain!(budget, key, limits, depth + 1)) + charge = _materializedsum(charge, + _writerowretain!(budget, item, limits, depth + 1)) + end + return charge + end + _reserveobjects!(budget) + return _MATERIALIZED_OBJECT_BYTES +end + +function _fillwriterowarrays!(columns::Vector{AbstractVector}, rows, + count::Int, limits::Limits, budget::_LiveByteBudget) + position = 0 + retained = Int64(0) + for row in rows + position += 1 + position <= count || throw(ArgumentError( + "row-oriented Tables source yielded more rows than declared")) + for index in eachindex(columns) + value = Tables.getcolumn(row, index) + retained = _materializedsum(retained, + _writerowretain!(budget, value, limits)) + columns[index][position] = value + end + end + position == count || throw(ArgumentError( + "row-oriented Tables source yielded $position rows but declared $count")) + return retained +end + +function _knownwriterowcolumns(rows, types, count::Int, limits::Limits, + budget::_LiveByteBudget) + start = _budgetused(budget) + try + charge = _materializedsum( + _materializedarraybytes(AbstractVector, length(types)), + _writerowarraycharge(types, count)) + _reserve!(budget, charge) + columns = _writerowarrays(types, count) + retained = _fillwriterowarrays!(columns, rows, count, limits, budget) + return columns, count, _materializedsum(charge, retained) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writerowgrowthcharge(types) + bytes = Int64(0) + for T in types + item = _materializedarraybytes(T, 1; header=false) + bytes = _materializedsum(bytes, + _materializedproduct(4, max(item, Int64(1)))) + end + return bytes +end + +function _pushwriterow!(columns::Vector{AbstractVector}, row, + limits::Limits, budget::_LiveByteBudget) + retained = Int64(0) + for index in eachindex(columns) + value = Tables.getcolumn(row, index) + retained = _materializedsum(retained, + _writerowretain!(budget, value, limits)) + push!(columns[index], value) + end + return retained +end + +function _copywriterowarrays(columns::Vector{AbstractVector}, types, + count::Int, budget::_LiveByteBudget) + outputcharge = _materializedsum( + _materializedarraybytes(AbstractVector, length(types)), + _writerowarraycharge(types, count)) + _reserve!(budget, outputcharge) + output = AbstractVector[] + sizehint!(output, length(types)) + for column in columns + push!(output, copy(column)) + end + return output, outputcharge +end + +function _unknownwriterowcolumns(rows, types, limits::Limits, + budget::_LiveByteBudget) + start = _budgetused(budget) + scratchcharge = Int64(0) + retainedcharge = Int64(0) + try + scratchcharge = _materializedsum(scratchcharge, + _reservearray!(budget, AbstractVector, length(types))) + headers = _writerowarraycharge(types, 0) + _reserve!(budget, headers) + scratchcharge = _materializedsum(scratchcharge, headers) + columns = _writerowarrays(types, 0) + growth = _writerowgrowthcharge(types) + count = 0 + for row in rows + requested = try + Base.checked_add(count, 1) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, requested, + limits.max_container_elements) + _reserve!(budget, growth) + scratchcharge = _materializedsum(scratchcharge, growth) + retainedcharge = _materializedsum(retainedcharge, + _pushwriterow!(columns, row, limits, budget)) + count = requested + end + output, outputcharge = _copywriterowarrays(columns, types, count, + budget) + _release!(budget, scratchcharge) + return output, count, + _materializedsum(outputcharge, retainedcharge) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writeinputrowcolumns(table, limits::Limits, + budget::_LiveByteBudget) + Base.@nospecialize table + rowsource = Tables.rows(table) + names, types = _writerowschema(rowsource, limits, budget) + start = _budgetused(budget) + try + normalized, namearraycharge = _validatewritecolumnnames(names, budget) + _preflightwriterowtypes(normalized, types, limits, budget) + count = _writerowcount(rowsource, limits) + columns, rowcount, rowcharge = count === nothing ? + _unknownwriterowcolumns(rowsource, types, limits, budget) : + _knownwriterowcolumns(rowsource, types, count, limits, budget) + paircharge = _reservearray!(budget, Pair{String,AbstractVector}, + length(columns)) + output = Pair{String,AbstractVector}[] + sizehint!(output, length(columns)) + for index in eachindex(columns, normalized) + push!(output, Pair{String,AbstractVector}( + normalized[index], columns[index])) + end + _release!(budget, namearraycharge) + return output, rowcount, _materializedsum(rowcharge, paircharge), nothing + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +struct _WriteColumnAccessValidator + table::Any + columns::Vector{Pair{String,AbstractVector}} +end + +function _writecolumnnameequal(raw::AbstractString, expected::String) + ncodeunits(raw) == ncodeunits(expected) || return false + rawunits = codeunits(raw) + expectedunits = codeunits(expected) + for index in eachindex(rawunits, expectedunits) + rawunits[index] == expectedunits[index] || return false + end + return true +end + +function _writecolumnnameequal(raw::Symbol, expected::String) + return String(raw) == expected +end + +function _writecolumnnameequal(raw, ::String) + _writecolumnnamebytes(raw) + return false +end + +function _validatewriteinput(::Nothing, ::_LiveByteBudget) + return +end + +function _validatewriteinput(validator::_WriteColumnAccessValidator, + budget::_LiveByteBudget) + start = _budgetused(budget) + try + columns = Tables.columns(validator.table) + names = Tables.columnnames(columns) + length(names) == length(validator.columns) || throw(ArgumentError( + "Parquet table column count changed during write")) + index = 0 + for raw in names + index += 1 + index <= length(validator.columns) || throw(ArgumentError( + "Parquet table column count changed during write")) + expected = validator.columns[index] + namebytes = _writecolumnnamebytes(raw) + temporary = _materializedsum(_MATERIALIZED_OBJECT_BYTES, namebytes) + _reserve!(budget, temporary) + matches = _writecolumnnameequal(raw, first(expected)) + _release!(budget, temporary) + matches || throw(ArgumentError( + "Parquet table column name or order changed during write")) + values = Tables.getcolumn(columns, raw) + values === last(expected) || throw(ArgumentError( + "Parquet table column identity changed during write")) + end + index == length(validator.columns) || throw(ArgumentError( + "Parquet table column count changed during write")) + return + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writeinputcolumns(table, limits::Limits, budget::_LiveByteBudget) + Base.@nospecialize table + Tables.columnaccess(table) || return _writeinputrowcolumns(table, + limits, budget) + columns = Tables.columns(table) + raw_names = Tables.columnnames(columns) + rawcharge = _reservearray!(budget, Any, length(raw_names)) + try + names = collect(raw_names) + isempty(names) && throw(ArgumentError( + "a Parquet table must have at least one column")) + normalized, namearraycharge = _validatewritecolumnnames(names, budget) + try + paircharge = _reservearray!(budget, + Pair{String,AbstractVector}, length(names)) + output = Pair{String,AbstractVector}[] + sizehint!(output, length(names)) + rows = nothing + for (raw, name) in zip(names, normalized) + values = Tables.getcolumn(columns, raw) + values isa AbstractVector || throw(ArgumentError( + "Parquet columns must be vectors")) + if rows === nothing + rows = length(values) + _checklimit(:container_elements, rows, + limits.max_container_elements) + end + length(values) == rows || throw(ArgumentError( + "Parquet columns have different lengths")) + push!(output, Pair{String,AbstractVector}(name, values)) + end + validatorcharge = _reserveobjects!(budget) + validator = _WriteColumnAccessValidator(table, output) + return output, something(rows, 0), + _materializedsum(paircharge, validatorcharge), validator + finally + _release!(budget, namearraycharge) + end + finally + _release!(budget, rawcharge) + end +end + +function _preflightwriteencoding(semantic::_NestedSchemaPlan, encoding, + dictionary::Bool, budget::_LiveByteBudget) + start = _budgetused(budget) + try + count = length(semantic.leaves) + choicecharge = _reservearray!(budget, WriteEncodingChoice, count) + _reservearray!(budget, WriteLeafPlan, count) + _reserveobjects!(budget, 3 * count + 2) + emptyschema = Metadata.SchemaElement[] + leaves = WriteLeafPlan[] + sizehint!(leaves, count) + for leaf in semantic.leaves + node = leaf.source + element = node.element + optional = element.repetition_type == + Metadata.FieldRepetitionType.OPTIONAL + column = WriteColumn(element.name, nothing, element.type_, + element.type_length, optional, element.logicalType, + element.converted_type, node.path, nothing, nothing, + node.max_repetition_level, node.max_definition_level, 0, + emptyschema) + push!(leaves, WriteLeafPlan(node.column_index, node.path, column)) + end + choices = _writeencodingchoices(leaves, encoding, dictionary) + used = _budgetused(budget) + temporary = used - start - choicecharge + temporary > 0 && _release!(budget, temporary) + return choices, choicecharge + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writefieldsencoded(table, limits::Limits, + budget::_LiveByteBudget, encoding, dictionary::Bool) + Base.@nospecialize table + start = _budgetused(budget) + try + columns, rows, inputcharge, validator = _writeinputcolumns(table, + limits, budget) + preflight = semantic -> _preflightwriteencoding(semantic, encoding, + dictionary, budget) + fields = _nestedwritefields(columns, rows, limits, budget; + preflight=preflight, sourcevalidator=validator) + _release!(budget, inputcharge) + return fields, rows + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writefields(table, limits::Limits, budget::_LiveByteBudget) + Base.@nospecialize table + return _writefieldsencoded(table, limits, budget, nothing, false) +end + +function _writefields(table, limits::Limits) + return _writefields(table, limits, _LiveByteBudget(limits)) +end + +function _presentvalues(column::WriteColumn, ::Type{T}) where {T} + output = T[] + sizehint!(output, length(column.values)) + for value in column.values + ismissing(value) && continue + push!(output, convert(T, value)) + end + return output +end + +function _plainbytearray(column::WriteColumn, limits::Limits) + output = UInt8[] + for value in column.values + ismissing(value) && continue + bytes = value isa AbstractString ? codeunits(value) : value + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + length(bytes) <= typemax(Int32) || throw(ArgumentError("byte array exceeds Int32 length")) + requested = Base.checked_add(length(output), Base.checked_add(4, length(bytes))) + _checklimit(:page_bytes, requested, limits.max_page_bytes) + _writelittle!(output, reinterpret(UInt32, Int32(length(bytes)))) + append!(output, bytes) + end + return output +end + +function _presentfixed(column::WriteColumn, limits::Limits) + width = column.type_length + width === nothing && throw(ArgumentError("fixed byte-array column has no width")) + fixedwidth = Int(width) + _checklimit(:string_bytes, fixedwidth, limits.max_string_bytes) + present = length(column.values) - count(ismissing, column.values) + total = Base.checked_mul(Int64(fixedwidth), Int64(present)) + _checklimit(:page_bytes, total, limits.max_page_bytes) + output = Matrix{UInt8}(undef, fixedwidth, present) + columnindex = 0 + for value in column.values + ismissing(value) && continue + _checkfixedvalue(value, width) + columnindex += 1 + @inbounds for byteindex in 1:fixedwidth + output[byteindex, columnindex] = value[byteindex] + end + end + return output +end + +function _plainpayload(column::WriteColumn, limits::Limits) + T = Base.nonmissingtype(eltype(column.values)) + if column.physical == Metadata.Type.BOOLEAN + return encode_plain(_presentvalues(column, Bool)) + elseif column.physical == Metadata.Type.INT32 + return encode_plain(_presentvalues(column, Int32)) + elseif column.physical == Metadata.Type.INT64 + return encode_plain(_presentvalues(column, Int64)) + elseif column.physical == Metadata.Type.FLOAT + return encode_plain(_presentvalues(column, Float32)) + elseif column.physical == Metadata.Type.DOUBLE + return encode_plain(_presentvalues(column, Float64)) + elseif column.physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY + return encode_plain_fixed(_presentfixed(column, limits)) + elseif T <: AbstractString || T <: AbstractVector{UInt8} + return _plainbytearray(column, limits) + end + throw(ArgumentError("unsupported PLAIN writer physical type $(column.physical)")) +end + +function _writeencoding(::Nothing) + return nothing +end + +function _writeencoding(encoding::Metadata.Encoding.T) + encoding == Metadata.Encoding.PLAIN && return encoding + encoding == Metadata.Encoding.DELTA_BINARY_PACKED && return encoding + encoding == Metadata.Encoding.DELTA_LENGTH_BYTE_ARRAY && return encoding + encoding == Metadata.Encoding.DELTA_BYTE_ARRAY && return encoding + encoding == Metadata.Encoding.BYTE_STREAM_SPLIT && return encoding + encoding == Metadata.Encoding.RLE && return encoding + encoding in (Metadata.Encoding.PLAIN_DICTIONARY, Metadata.Encoding.RLE_DICTIONARY) && + throw(ArgumentError( + "dictionary encodings cannot be selected directly; use :dictionary")) + encoding == Metadata.Encoding.BIT_PACKED && + throw(ArgumentError("deprecated BIT_PACKED encoding is read-only")) + throw(ArgumentError("unsupported explicit Parquet value encoding $encoding")) +end + +function _writeencoding(encoding::Symbol) + name = Symbol(lowercase(String(encoding))) + name === :plain && return Metadata.Encoding.PLAIN + name === :delta_binary_packed && return Metadata.Encoding.DELTA_BINARY_PACKED + name === :delta_length_byte_array && return Metadata.Encoding.DELTA_LENGTH_BYTE_ARRAY + name === :delta_byte_array && return Metadata.Encoding.DELTA_BYTE_ARRAY + name === :byte_stream_split && return Metadata.Encoding.BYTE_STREAM_SPLIT + name === :rle && return Metadata.Encoding.RLE + name in (:plain_dictionary, :rle_dictionary) && + throw(ArgumentError( + "dictionary encodings cannot be selected directly; use :dictionary")) + name === :bit_packed && + throw(ArgumentError("deprecated BIT_PACKED encoding is read-only")) + throw(ArgumentError("unknown explicit Parquet value encoding $(repr(encoding))")) +end + +function _writeencoding(encoding::AbstractString) + return _writeencoding(Symbol(encoding)) +end + +function _writeencoding(encoding) + throw(ArgumentError( + "explicit Parquet value encoding must be a Symbol, string, Encoding value, or nothing")) +end + +function _writeencodingchoice(encoding::Metadata.Encoding.T) + return WriteEncodingChoice(_writeencoding(encoding), false) +end + +function _writeencodingchoice(encoding::Symbol) + Symbol(lowercase(String(encoding))) === :dictionary && + return WriteEncodingChoice(nothing, true) + return WriteEncodingChoice(_writeencoding(encoding), false) +end + +function _writeencodingchoice(encoding::AbstractString) + return _writeencodingchoice(Symbol(encoding)) +end + +function _writeencodingchoice(::Nothing) + throw(ArgumentError( + "a per-column Parquet encoding cannot be nothing; omit the column override")) +end + +function _writeencodingchoice(encoding) + throw(ArgumentError( + "a Parquet encoding must be a Symbol, string, or Encoding value, got $(typeof(encoding))")) +end + +function _writeencodingcolumn(name::Symbol) + return String(name) +end + +function _writeencodingcolumn(name::AbstractString) + return String(name) +end + +function _writeencodingcolumn(name) + throw(ArgumentError( + "Parquet encoding policy keys must be Symbols or strings, got $(typeof(name))")) +end + +function _writeencodingoverrides(entries) + overrides = Dict{String,WriteEncodingChoice}() + for (rawname, encoding) in entries + name = _writeencodingcolumn(rawname) + haskey(overrides, name) && + throw(ArgumentError("duplicate Parquet encoding policy for column $(repr(name))")) + overrides[name] = _writeencodingchoice(encoding) + end + return overrides +end + +function _validatewriteencodingchoice(column::WriteColumn, choice::WriteEncodingChoice) + encoding = choice.encoding + encoding === nothing || _validatewriteencoding(column, encoding) + return +end + +function _mappedwriteencodingchoices(columns::Vector{WriteColumn}, entries, dictionary::Bool) + overrides = _writeencodingoverrides(entries) + names = Set(column.name for column in columns) + unknown = sort!(String[name for name in keys(overrides) if !(name in names)]) + isempty(unknown) || throw(ArgumentError( + "unknown Parquet writer column" * (length(unknown) == 1 ? " " : "s ") * + join(repr.(unknown), ", ") * " in encoding policy")) + default = WriteEncodingChoice(nothing, dictionary) + choices = WriteEncodingChoice[] + sizehint!(choices, length(columns)) + for column in columns + choice = get(overrides, column.name, default) + _validatewriteencodingchoice(column, choice) + push!(choices, choice) + end + return choices +end + +function _writeencodingchoices(columns::Vector{WriteColumn}, ::Nothing, dictionary::Bool) + return fill(WriteEncodingChoice(nothing, dictionary), length(columns)) +end + +function _writeencodingchoices(columns::Vector{WriteColumn}, encoding::Union{ + Symbol,AbstractString,Metadata.Encoding.T}, dictionary::Bool) + dictionary && throw(ArgumentError( + "dictionary=true conflicts with a table-wide encoding; use :dictionary or a per-column mapping")) + choice = _writeencodingchoice(encoding) + foreach(column -> _validatewriteencodingchoice(column, choice), columns) + return fill(choice, length(columns)) +end + +function _writeencodingchoices(columns::Vector{WriteColumn}, encoding::Pair, dictionary::Bool) + return _mappedwriteencodingchoices(columns, (encoding,), dictionary) +end + +function _writeencodingchoices(columns::Vector{WriteColumn}, encoding::NamedTuple, + dictionary::Bool) + return _mappedwriteencodingchoices(columns, pairs(encoding), dictionary) +end + +function _writeencodingchoices(columns::Vector{WriteColumn}, encoding::AbstractDict, + dictionary::Bool) + return _mappedwriteencodingchoices(columns, pairs(encoding), dictionary) +end + +function _writeencodingchoices(::Vector{WriteColumn}, encoding, ::Bool) + throw(ArgumentError( + "Parquet encoding must be nothing, a Symbol, string, Pair, NamedTuple, or AbstractDict, got $(typeof(encoding))")) +end + +function _writeencodingselector(selector::Integer) + selector isa Bool && throw(ArgumentError( + "a Parquet physical leaf ordinal must be a positive integer")) + selector > 0 || throw(ArgumentError( + "a Parquet physical leaf ordinal must be positive, got $selector")) + return (:ordinal, selector) +end + +function _writeencodingselector(selector::Tuple) + isempty(selector) && throw(ArgumentError( + "a Parquet physical leaf path selector cannot be empty")) + path = String[] + sizehint!(path, length(selector)) + for segment in selector + segment isa Union{Symbol,AbstractString} || throw(ArgumentError( + "Parquet physical leaf path segments must be Symbols or strings, got $(typeof(segment))")) + push!(path, String(segment)) + end + return (:path, Tuple(path)) +end + +function _writeencodingselector(selector::Symbol) + return (:name, String(selector)) +end + +function _writeencodingselector(selector::AbstractString) + return (:name, String(selector)) +end + +function _writeencodingselector(selector) + throw(ArgumentError( + "a Parquet leaf encoding selector must be a positive integer, exact path tuple, Symbol, or string, got $(typeof(selector))")) +end + +function _writepathmatches(path::Vector{String}, selector::Tuple) + length(path) == length(selector) || return false + for (pathsegment, selectorsegment) in zip(path, selector) + pathsegment == selectorsegment || return false + end + return true +end + +function _singlewriteencodingmatch(matches::Vector{Int}, selector; kind::String) + isempty(matches) && throw(ArgumentError( + "unknown Parquet writer $kind $(repr(selector)) in encoding policy")) + length(matches) == 1 || throw(ArgumentError( + "ambiguous Parquet writer $kind $(repr(selector)) selects multiple physical leaves; use a positive leaf ordinal")) + return only(matches) +end + +function _writeencodingleafindex(leaves::Vector{WriteLeafPlan}, + selector::Tuple{Symbol,<:Integer}) + ordinal = selector[2] + matches = Int[] + for (index, leaf) in enumerate(leaves) + leaf.ordinal == ordinal && push!(matches, index) + end + return _singlewriteencodingmatch(matches, ordinal; + kind="physical leaf ordinal") +end + +function _writeencodingleafindex(leaves::Vector{WriteLeafPlan}, + selector::Tuple{Symbol,<:Tuple}) + path = selector[2] + matches = Int[] + for (index, leaf) in enumerate(leaves) + _writepathmatches(leaf.path, path) && push!(matches, index) + end + return _singlewriteencodingmatch(matches, path; + kind="physical leaf path") +end + +function _writeencodingleafindex(leaves::Vector{WriteLeafPlan}, + selector::Tuple{Symbol,String}) + name = selector[2] + flat = Int[] + for (index, leaf) in enumerate(leaves) + length(leaf.path) == 1 && only(leaf.path) == name && push!(flat, index) + end + isempty(flat) || return _singlewriteencodingmatch(flat, name; + kind="column") + group = Int[] + for (index, leaf) in enumerate(leaves) + !isempty(leaf.path) && first(leaf.path) == name && push!(group, index) + end + return _singlewriteencodingmatch(group, name; + kind="top-level field") +end + +function _mappedwriteencodingchoices(leaves::Vector{WriteLeafPlan}, entries, + dictionary::Bool) + selectors = Set{Any}() + assigned = Set{Int}() + overrides = Dict{Int,WriteEncodingChoice}() + for (rawselector, encoding) in entries + selector = _writeencodingselector(rawselector) + selector in selectors && throw(ArgumentError( + "duplicate Parquet encoding policy selector $(repr(rawselector))")) + push!(selectors, selector) + index = _writeencodingleafindex(leaves, selector) + index in assigned && throw(ArgumentError( + "multiple Parquet encoding policy selectors assign physical leaf ordinal $(leaves[index].ordinal)")) + choice = _writeencodingchoice(encoding) + _validatewriteencodingchoice(leaves[index].column, choice) + push!(assigned, index) + overrides[index] = choice + end + default = WriteEncodingChoice(nothing, dictionary) + choices = WriteEncodingChoice[] + sizehint!(choices, length(leaves)) + for index in eachindex(leaves) + push!(choices, get(overrides, index, default)) + end + return choices +end + +function _writeencodingchoices(leaves::Vector{WriteLeafPlan}, ::Nothing, + dictionary::Bool) + return fill(WriteEncodingChoice(nothing, dictionary), length(leaves)) +end + +function _writeencodingchoices(leaves::Vector{WriteLeafPlan}, encoding::Union{ + Symbol,AbstractString,Metadata.Encoding.T}, dictionary::Bool) + dictionary && throw(ArgumentError( + "dictionary=true conflicts with a table-wide encoding; use :dictionary or a per-column mapping")) + choice = _writeencodingchoice(encoding) + foreach(leaf -> _validatewriteencodingchoice(leaf.column, choice), leaves) + return fill(choice, length(leaves)) +end + +function _writeencodingchoices(leaves::Vector{WriteLeafPlan}, encoding::Pair, + dictionary::Bool) + return _mappedwriteencodingchoices(leaves, (encoding,), dictionary) +end + +function _writeencodingchoices(leaves::Vector{WriteLeafPlan}, encoding::NamedTuple, + dictionary::Bool) + return _mappedwriteencodingchoices(leaves, pairs(encoding), dictionary) +end + +function _writeencodingchoices(leaves::Vector{WriteLeafPlan}, encoding::AbstractDict, + dictionary::Bool) + return _mappedwriteencodingchoices(leaves, pairs(encoding), dictionary) +end + +function _writeencodingchoices(::Vector{WriteLeafPlan}, encoding, ::Bool) + throw(ArgumentError( + "Parquet encoding must be nothing, a Symbol, string, Pair, NamedTuple, or AbstractDict, got $(typeof(encoding))")) +end + +function _validatewritechoicecount(leaves::Vector{WriteLeafPlan}, + choices::Vector{WriteEncodingChoice}) + length(choices) == length(leaves) || throw(UnsupportedFeatureError( + "nested fields with multiple physical leaves need per-leaf encoding selection")) + return +end + +function _validatewriteencoding(column::WriteColumn, encoding::Metadata.Encoding.T) + physical = column.physical + encoding == Metadata.Encoding.PLAIN && return + encoding == Metadata.Encoding.DELTA_BINARY_PACKED && + physical in (Metadata.Type.INT32, Metadata.Type.INT64) && return + encoding in (Metadata.Encoding.DELTA_LENGTH_BYTE_ARRAY, Metadata.Encoding.DELTA_BYTE_ARRAY) && + physical == Metadata.Type.BYTE_ARRAY && return + encoding == Metadata.Encoding.DELTA_BYTE_ARRAY && + physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY && return + encoding == Metadata.Encoding.BYTE_STREAM_SPLIT && + physical in (Metadata.Type.INT32, Metadata.Type.INT64, Metadata.Type.FLOAT, + Metadata.Type.DOUBLE, Metadata.Type.FIXED_LEN_BYTE_ARRAY) && return + encoding == Metadata.Encoding.RLE && physical == Metadata.Type.BOOLEAN && return + throw(ArgumentError( + "Parquet value encoding $encoding is not valid for column $(repr(column.name)) " * + "with physical type $physical")) +end + +function _presentbytearrays(column::WriteColumn, limits::Limits) + T = Base.nonmissingtype(eltype(column.values)) + output = T[] + sizehint!(output, length(column.values)) + total = Int64(0) + for value in column.values + ismissing(value) && continue + bytes = value isa AbstractString ? codeunits(value) : value + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + length(bytes) <= typemax(Int32) || throw(ArgumentError("byte array exceeds Int32 length")) + total = try + Base.checked_add(total, Int64(length(bytes))) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:page_bytes, typemax(Int64), limits.max_page_bytes)) + end + _checklimit(:page_bytes, total, limits.max_page_bytes) + push!(output, value) + end + return output +end + +function _encodedpayload(column::WriteColumn, encoding::Metadata.Encoding.T, limits::Limits) + _validatewriteencoding(column, encoding) + payload = if encoding == Metadata.Encoding.PLAIN + _plainpayload(column, limits) + elseif encoding == Metadata.Encoding.DELTA_BINARY_PACKED + T = column.physical == Metadata.Type.INT32 ? Int32 : Int64 + encode_delta_binary_packed(_presentvalues(column, T)) + elseif encoding == Metadata.Encoding.DELTA_LENGTH_BYTE_ARRAY + encode_delta_length_byte_array(_presentbytearrays(column, limits)) + elseif encoding == Metadata.Encoding.DELTA_BYTE_ARRAY + column.physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY ? + encode_delta_byte_array_fixed(_presentfixed(column, limits)) : + encode_delta_byte_array(_presentbytearrays(column, limits)) + elseif encoding == Metadata.Encoding.BYTE_STREAM_SPLIT + if column.physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY + encode_byte_stream_split_fixed(_presentfixed(column, limits)) + else + T = column.physical == Metadata.Type.INT32 ? Int32 : + column.physical == Metadata.Type.INT64 ? Int64 : + column.physical == Metadata.Type.FLOAT ? Float32 : Float64 + encode_byte_stream_split(_presentvalues(column, T)) + end + else + values = UInt64[value ? 1 : 0 for value in _presentvalues(column, Bool)] + encode_hybrid(values, 1; length_prefix=true) + end + _checklimit(:page_bytes, length(payload), limits.max_page_bytes) + length(payload) <= typemax(Int32) || + throw(ArgumentError("encoded Parquet values exceed Int32 bytes")) + return payload +end + +function _columnentrycount(column::WriteColumn) + column.definitions === nothing && return length(column.values) + return length(column.definitions) +end + +function _writerpresentcount(column::WriteColumn) + return length(column.values) - count(ismissing, column.values) +end + +function _writerrawpayloadbytes(column::WriteColumn) + present = _writerpresentcount(column) + physical = column.physical + physical == Metadata.Type.BOOLEAN && return Int64(cld(present, 8)) + physical in (Metadata.Type.INT32, Metadata.Type.FLOAT) && + return _materializedproduct(present, 4) + physical in (Metadata.Type.INT64, Metadata.Type.DOUBLE) && + return _materializedproduct(present, 8) + if physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY + width = column.type_length + width === nothing && throw(ArgumentError( + "fixed byte-array column has no width")) + return _materializedproduct(present, width) + end + physical == Metadata.Type.BYTE_ARRAY || return Int64(0) + bytes = Int64(0) + for value in column.values + ismissing(value) && continue + payload = value isa AbstractString ? ncodeunits(value) : length(value) + bytes = _materializedsum(bytes, _materializedsum(4, payload)) + end + return bytes +end + +function _writerpageworkingbytes(column::WriteColumn, dictionary::Bool) + entries = _columnentrycount(column) + raw = _writerrawpayloadbytes(column) + encoded = _materializedsum(raw, _materializedsum( + _materializedproduct(entries, 24), 2048)) + factor = dictionary && column.physical != Metadata.Type.BOOLEAN ? 16 : 8 + bytes = _materializedproduct(encoded, factor) + bytes = _materializedsum(bytes, + _materializedproduct(entries, _MATERIALIZED_OBJECT_BYTES)) + return _materializedsum(bytes, + _materializedproduct(8, _MATERIALIZED_OBJECT_BYTES)) +end + +function _columnpageslivebytes(pages::ColumnPages) + bytes = _materializedarraybytes(UInt8, length(pages.bytes)) + bytes = _materializedsum(bytes, + _materializedarraybytes(Metadata.Encoding.T, length(pages.encodings))) + bytes = _materializedsum(bytes, + _materializedarraybytes(Metadata.PageEncodingStats, + length(pages.encoding_stats))) + bytes = _materializedsum(bytes, + _materializedarraybytes(Metadata.PageLocation, + length(pages.page_locations))) + return _materializedsum(bytes, + _materializedproduct(3, _MATERIALIZED_OBJECT_BYTES)) +end + +function _columnpagestransferredbytes(pages::ColumnPages) + bytes = _materializedarraybytes(Metadata.Encoding.T, + length(pages.encodings)) + return _materializedsum(bytes, + _materializedarraybytes(Metadata.PageEncodingStats, + length(pages.encoding_stats))) +end + +function _columnnullcount(column::WriteColumn) + column.definitions === nothing && return count(ismissing, column.values) + maximum = UInt64(column.max_definition_level) + return count(!=(maximum), column.definitions) +end + +function _repetitionpayload(column::WriteColumn; length_prefix::Bool=true) + iszero(column.max_repetition_level) && return UInt8[] + levels = something(column.repetitions) + return encode_hybrid(levels, _levelbitwidth(column.max_repetition_level); + length_prefix=length_prefix) +end + +function _definitionpayload(column::WriteColumn; length_prefix::Bool=true) + iszero(column.max_definition_level) && return UInt8[] + levels = column.definitions === nothing ? + UInt64[ismissing(value) ? 0 : 1 for value in column.values] : + column.definitions + return encode_hybrid(levels, _levelbitwidth(column.max_definition_level); + length_prefix=length_prefix) +end + +function _writecodec(codec::Metadata.CompressionCodec.T) + codecwritable(codec) || _unwritablecodec(codec) + return codec +end + +function _writecodec(codec::Symbol) + name = Symbol(lowercase(String(codec))) + name === :uncompressed && return Metadata.CompressionCodec.UNCOMPRESSED + name === :snappy && return Metadata.CompressionCodec.SNAPPY + name === :gzip && return Metadata.CompressionCodec.GZIP + name === :brotli && return Metadata.CompressionCodec.BROTLI + name === :zstd && return Metadata.CompressionCodec.ZSTD + name === :lz4_raw && return Metadata.CompressionCodec.LZ4_RAW + name === :lz4 && return _writecodec(Metadata.CompressionCodec.LZ4) + name === :lzo && return _writecodec(Metadata.CompressionCodec.LZO) + throw(ArgumentError("unknown Parquet compression codec $(repr(codec))")) +end + +function _writecodec(codec::AbstractString) + return _writecodec(Symbol(codec)) +end + +function _writecodec(codec) + throw(ArgumentError("Parquet compression codec must be a Symbol, string, or CompressionCodec value")) +end + +function _writepageversion(pageversion::Symbol) + version = Symbol(lowercase(String(pageversion))) + version === :v1 && return version + version === :v2 && return version + throw(ArgumentError("Parquet page version must be :v1 or :v2")) +end + +function _writepageversion(pageversion::AbstractString) + return _writepageversion(Symbol(pageversion)) +end + +function _writepageversion(pageversion) + throw(ArgumentError("Parquet page version must be :v1 or :v2")) +end + +function _framedpage(payload::Vector{UInt8}, type::Metadata.PageType.T, limits::Limits; + checksum::Bool, codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, data_header=nothing, dictionary_header=nothing) + _checklimit(:page_bytes, length(payload), limits.max_page_bytes) + length(payload) <= typemax(Int32) || throw(ArgumentError("Parquet page exceeds Int32 bytes")) + encoded = compress(codec, payload; level=compressionlevel) + _checklimit(:page_bytes, length(encoded), limits.max_page_bytes) + length(encoded) <= typemax(Int32) || throw(ArgumentError("compressed Parquet page exceeds Int32 bytes")) + crc = checksum ? reinterpret(Int32, pagechecksum(encoded)) : nothing + header = Metadata.PageHeader( + type_=type, + uncompressed_page_size=Int32(length(payload)), + compressed_page_size=Int32(length(encoded)), + crc=crc, + data_page_header=data_header, + dictionary_page_header=dictionary_header, + ) + headerbytes = Thrift.encode(header) + _checklimit(:page_header_bytes, length(headerbytes), limits.max_page_header_bytes) + return vcat(headerbytes, encoded), length(headerbytes), length(payload) +end + +function _framedpagev2(column::WriteColumn, values::Vector{UInt8}, + encoding::Metadata.Encoding.T, limits::Limits; checksum::Bool, + codec::Metadata.CompressionCodec.T, compressionlevel::Union{Nothing,Integer}) + repetition = _repetitionpayload(column; length_prefix=false) + definition = _definitionpayload(column; length_prefix=false) + candidate = compress(codec, values; level=compressionlevel) + compressedvalues = codec != Metadata.CompressionCodec.UNCOMPRESSED && + !isempty(values) && length(candidate) < length(values) + encoded = compressedvalues ? candidate : copy(values) + uncompressed = Int64(length(repetition)) + Int64(length(definition)) + + Int64(length(values)) + compressed = Int64(length(repetition)) + Int64(length(definition)) + + Int64(length(encoded)) + _checklimit(:page_bytes, uncompressed, limits.max_page_bytes) + _checklimit(:page_bytes, compressed, limits.max_page_bytes) + uncompressed <= typemax(Int32) || throw(ArgumentError("Parquet page exceeds Int32 bytes")) + compressed <= typemax(Int32) || throw(ArgumentError("compressed Parquet page exceeds Int32 bytes")) + payload = vcat(repetition, definition, encoded) + crc = checksum ? reinterpret(Int32, pagechecksum(payload)) : nothing + data_header = Metadata.DataPageHeaderV2( + num_values=Int32(_columnentrycount(column)), + num_nulls=Int32(_columnnullcount(column)), + num_rows=Int32(column.rows), + encoding=encoding, + definition_levels_byte_length=Int32(length(definition)), + repetition_levels_byte_length=Int32(length(repetition)), + is_compressed=compressedvalues, + ) + header = Metadata.PageHeader( + type_=Metadata.PageType.DATA_PAGE_V2, + uncompressed_page_size=Int32(uncompressed), + compressed_page_size=Int32(compressed), + crc=crc, + data_page_header_v2=data_header, + ) + headerbytes = Thrift.encode(header) + _checklimit(:page_header_bytes, length(headerbytes), limits.max_page_header_bytes) + return vcat(headerbytes, payload), length(headerbytes), Int(uncompressed) +end + +function _datapagebytes(column::WriteColumn, values::Vector{UInt8}, + encoding::Metadata.Encoding.T, pageversion::Symbol, limits::Limits; checksum::Bool, + codec::Metadata.CompressionCodec.T, compressionlevel::Union{Nothing,Integer}) + if pageversion === :v2 + return _framedpagev2(column, values, encoding, limits; checksum=checksum, + codec=codec, compressionlevel=compressionlevel) + end + payload = _repetitionpayload(column) + append!(payload, _definitionpayload(column)) + append!(payload, values) + data_header = Metadata.DataPageHeader( + num_values=Int32(_columnentrycount(column)), + encoding=encoding, + definition_level_encoding=Metadata.Encoding.RLE, + repetition_level_encoding=Metadata.Encoding.RLE, + ) + return _framedpage(payload, Metadata.PageType.DATA_PAGE, limits; + checksum=checksum, codec=codec, compressionlevel=compressionlevel, + data_header=data_header) +end + +function _datapagetype(pageversion::Symbol) + pageversion === :v1 && return Metadata.PageType.DATA_PAGE + return Metadata.PageType.DATA_PAGE_V2 +end + +function _schemaelements(column::WriteColumn) + return column.schema +end + +function _withoutcolumnschema(column::WriteColumn, path::Vector{String}=column.path) + return WriteColumn(column.name, column.values, column.physical, column.type_length, + column.optional, column.logical, column.converted, copy(path), column.repetitions, + column.definitions, column.max_repetition_level, column.max_definition_level, + column.rows, Metadata.SchemaElement[]) +end + +function _writefieldplan(column::WriteColumn) + schema = _schemaelements(column) + isempty(schema) && throw(ArgumentError( + "writer column $(repr(column.name)) has no top-level field schema")) + return WriteFieldPlan(copy(schema), WriteColumn[_withoutcolumnschema(column)]) +end + +function _writeschemaelementcount(fields::Vector{WriteFieldPlan}, limits::Limits) + total = 1 + for field in fields + isempty(field.schema) && throw(ArgumentError("writer field has an empty schema")) + isempty(field.leaves) && throw(ArgumentError("writer field has no physical leaves")) + total = try + Base.checked_add(total, length(field.schema)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, total, limits.max_container_elements) + end + return total +end + +function _flattenwriteschema(fields::Vector{WriteFieldPlan}, limits::Limits, + budget::_LiveByteBudget) + isempty(fields) && throw(ArgumentError("a Parquet table must have at least one column")) + length(fields) <= typemax(Int32) || + throw(ArgumentError("Parquet schema has more than Int32 top-level fields")) + count = _writeschemaelementcount(fields, limits) + _reservearray!(budget, Metadata.SchemaElement, count) + elements = Metadata.SchemaElement[ + Metadata.SchemaElement(name="schema", num_children=Int32(length(fields))), + ] + sizehint!(elements, count) + for field in fields + append!(elements, field.schema) + end + return elements +end + +function _writeschemaleafcount(node::SchemaNode) + node.element.type_ !== nothing && return 1 + total = 0 + for child in node.children + total = Base.checked_add(total, _writeschemaleafcount(child)) + end + return total +end + +function _validatewriteleaf(node::SchemaNode, column::WriteColumn, rows::Int) + isempty(column.schema) || throw(ArgumentError( + "row-group leaf $(repr(column.name)) still owns a schema fragment")) + column.rows == rows || throw(ArgumentError( + "row-group leaf $(repr(column.name)) has $(column.rows) rows but $rows were expected")) + column.path == node.path || throw(ArgumentError( + "writer leaf path $(repr(column.path)) does not match schema path $(repr(node.path))")) + column.physical == node.element.type_ || throw(ArgumentError( + "writer leaf $(repr(node.path)) has a physical type that does not match its schema")) + column.type_length == node.element.type_length || throw(ArgumentError( + "writer leaf $(repr(node.path)) has a type length that does not match its schema")) + column.logical == node.element.logicalType || throw(ArgumentError( + "writer leaf $(repr(node.path)) has a logical type that does not match its schema")) + column.converted == node.element.converted_type || throw(ArgumentError( + "writer leaf $(repr(node.path)) has a converted type that does not match its schema")) + column.max_repetition_level == node.max_repetition_level || throw(ArgumentError( + "writer leaf $(repr(node.path)) has a repetition level that does not match its schema")) + column.max_definition_level == node.max_definition_level || throw(ArgumentError( + "writer leaf $(repr(node.path)) has a definition level that does not match its schema")) + optional = node.element.repetition_type == Metadata.FieldRepetitionType.OPTIONAL + column.optional == optional || throw(ArgumentError( + "writer leaf $(repr(node.path)) has optionality that does not match its schema")) + return +end + +function _writeleafplanbytes(nodes) + bytes = _materializedarraybytes(WriteLeafPlan, length(nodes)) + for node in nodes + bytes = _materializedsum(bytes, + _materializedarraybytes(String, length(node.path))) + bytes = _materializedsum(bytes, _MATERIALIZED_OBJECT_BYTES) + end + return bytes +end + +function _writeplanleaves(fields::Vector{WriteFieldPlan}, schema::Schema, rows::Int, + limits::Limits, budget::_LiveByteBudget) + start = _budgetused(budget) + length(fields) == length(schema.root.children) || throw(ArgumentError( + "writer top-level fields do not match the parsed schema")) + columncharge = _reservearray!(budget, WriteColumn, length(schema.leaves)) + try + columns = WriteColumn[] + sizehint!(columns, length(schema.leaves)) + for (field, node) in zip(fields, schema.root.children) + length(field.leaves) == _writeschemaleafcount(node) || + throw(ArgumentError( + "writer field $(repr(node.element.name)) does not own its schema leaves")) + append!(columns, field.leaves) + end + length(columns) == length(schema.leaves) || throw(ArgumentError( + "writer physical leaves do not match the parsed schema")) + leafcharge = _writeleafplanbytes(schema.leaves) + _reserve!(budget, leafcharge) + leaves = WriteLeafPlan[] + sizehint!(leaves, length(columns)) + for (index, (node, column)) in enumerate(zip(schema.leaves, columns)) + prepared, entries, dense, payload, preparedcharge = + _writeprepareleaf(node, column, rows, limits, budget) + leafcharge = _materializedsum(leafcharge, preparedcharge) + _validatewriteleaf(node, prepared, rows) + ordinal = Int32(index) + node.column_index == ordinal || throw(ArgumentError( + "writer leaf ordinal does not match the parsed schema")) + push!(leaves, WriteLeafPlan(ordinal, copy(node.path), prepared, + entries, dense, payload)) + end + _release!(budget, columncharge) + return leaves, leafcharge + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writeplan(fields::Vector{WriteFieldPlan}, rows::Int, limits::Limits, + budget::_LiveByteBudget) + start = _budgetused(budget) + try + rows >= 0 || throw(ArgumentError( + "Parquet row count must be nonnegative")) + elements = _flattenwriteschema(fields, limits, budget) + schema = Schema(elements; limits=limits, budget=budget) + leaves, leafcharge = _writeplanleaves(fields, schema, rows, limits, + budget) + _reservearray!(budget, WriteRowGroupPlan, iszero(rows) ? 0 : 1) + _reserveobjects!(budget, 2) + rowgroups = iszero(rows) ? WriteRowGroupPlan[] : + WriteRowGroupPlan[WriteRowGroupPlan(rows, leaves)] + plan = WritePlan(elements, schema, rows, rowgroups) + iszero(rows) && _release!(budget, leafcharge) + return plan + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writeplan(fields::Vector{WriteFieldPlan}, rows::Int, limits::Limits) + return _writeplan(fields, rows, limits, _LiveByteBudget(limits)) +end + +function _writefieldplan(column::WriteColumn, budget::_LiveByteBudget) + schema = _schemaelements(column) + isempty(schema) && throw(ArgumentError( + "writer column $(repr(column.name)) has no top-level field schema")) + _reservearray!(budget, Metadata.SchemaElement, length(schema)) + _reservearray!(budget, WriteColumn, 1) + _reservearray!(budget, String, length(column.path)) + _reserveobjects!(budget, 2) + return WriteFieldPlan(copy(schema), WriteColumn[_withoutcolumnschema(column)]) +end + +function _writeplan(columns::Vector{WriteColumn}, rows::Int, limits::Limits, + budget::_LiveByteBudget) + start = _budgetused(budget) + try + _reservearray!(budget, WriteFieldPlan, length(columns)) + fields = WriteFieldPlan[] + sizehint!(fields, length(columns)) + for column in columns + push!(fields, _writefieldplan(column, budget)) + end + return _writeplan(fields, rows, limits, budget) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writeplan(columns::Vector{WriteColumn}, rows::Int, limits::Limits) + return _writeplan(columns, rows, limits, _LiveByteBudget(limits)) +end + +function _writeplan(table, limits::Limits, budget::_LiveByteBudget) + start = _budgetused(budget) + try + fields, rows = _writefields(table, limits, budget) + return _writeplan(fields, rows, limits, budget) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writeplan(table, limits::Limits) + return _writeplan(table, limits, _LiveByteBudget(limits)) +end + +function _writeplan(table) + return _writeplan(table, Limits()) +end + +function _columnchunk(leaf::WriteLeafPlan, offset::Int64, pages::ColumnPages, + codec::Metadata.CompressionCodec.T, + statistics::Union{Nothing,Metadata.Statistics}=nothing) + column = leaf.column + dictionary_offset = pages.dictionary_offset === nothing ? nothing : + offset + pages.dictionary_offset + compressed = Int64(length(pages.bytes)) + metadata = Metadata.ColumnMetaData( + type_=column.physical, + encodings=pages.encodings, + path_in_schema=leaf.path, + codec=codec, + num_values=Int64(_columnentrycount(column)), + total_uncompressed_size=pages.uncompressed_size, + total_compressed_size=compressed, + data_page_offset=offset + pages.data_offset, + dictionary_page_offset=dictionary_offset, + statistics=statistics, + encoding_stats=pages.encoding_stats, + ) + return Metadata.ColumnChunk(file_offset=Int64(0), meta_data=metadata) +end + +function _addgroupsize(total::Int64, value::Int64) + return try + Base.checked_add(total, value) + catch err + err isa OverflowError || rethrow() + throw(ArgumentError("Parquet row group byte size exceeds Int64")) + end +end + +const _WRITE_FOOTER_CONTROL_BYTES = + _materializedproduct(2, _MATERIALIZED_OBJECT_BYTES) + +function _writeencodefooter(value, limits::Limits, budget::_LiveByteBudget) + start = _budgetused(budget) + try + _reserve!(budget, _WRITE_FOOTER_CONTROL_BYTES) + exact = try + Thrift._encodedsize(value) + finally + _release!(budget, _WRITE_FOOTER_CONTROL_BYTES) + end + exact <= typemax(UInt32) || throw(ArgumentError( + "Parquet footer exceeds UInt32 bytes")) + _checklimit(:footer_bytes, exact, limits.max_footer_bytes) + _reserve!(budget, _WRITE_FOOTER_CONTROL_BYTES) + charge = _reservearray!(budget, UInt8, exact) + bytes = Vector{UInt8}(undef, Int(exact)) + Thrift._encodefixed!(bytes, value) + _release!(budget, _WRITE_FOOTER_CONTROL_BYTES) + return bytes, charge + catch err + used = _budgetused(budget) + used > start && _release!(budget, used - start) + err isa Thrift._WriteCountOverflow && throw(ArgumentError( + "Parquet footer exceeds UInt32 bytes")) + rethrow() + end +end + +function _reserveoutputgrowth!(budget::_LiveByteBudget, bytes::Integer) + charge = _materializedproduct(bytes, 2) + _reserve!(budget, charge) + return charge +end + +function _writekeyvaluemetadata(table, ::Limits, ::_LiveByteBudget) + Base.@nospecialize table + return nothing +end + +function _encodefile(table; checksum::Bool=true, dictionary::Bool=false, + codec=:uncompressed, compressionlevel::Union{Nothing,Integer}=nothing, + pageversion=:v1, encoding=nothing, rowgroupsize=1_048_576, + pagesize=1024 * 1024, pageindex::Bool=true, statistics::Bool=true, + limits::Limits=Limits()) + Base.@nospecialize table + statisticslimit = _statisticlimit(limits) + budget = _LiveByteBudget(limits) + _reserveobjects!(budget, 2) + compression = _writecodec(codec) + version = _writepageversion(pageversion) + target = _writepagesize(pagesize) + _writerowgroupsize(rowgroupsize, 0) + fields, rows = _writefieldsencoded(table, limits, budget, encoding, + dictionary) + initialplan = _writeplan(fields, rows, limits, budget) + selectorcharge = Int64(0) + leaves = if isempty(initialplan.rowgroups) + selected, charge = _writeplanleaves(fields, initialplan.schema, rows, limits, + budget) + selectorcharge = charge + selected + else + only(initialplan.rowgroups).leaves + end + _reservearray!(budget, WriteEncodingChoice, length(leaves)) + _reserveobjects!(budget, 4 * length(leaves) + 1) + choices = try + _writeencodingchoices(leaves, encoding, dictionary) + finally + iszero(selectorcharge) || _release!(budget, selectorcharge) + end + plan = _splitwriteplan(initialplan, rowgroupsize, limits, budget) + leaves = nothing + initialplan = nothing + columnorders = statistics ? _writecolumnorders(plan.schema, budget) : nothing + _reservearray!(budget, UInt8, length(PARQUET_MAGIC)) + output = copy(PARQUET_MAGIC) + _reservearray!(budget, Metadata.RowGroup, length(plan.rowgroups)) + rowgroups = Metadata.RowGroup[] + sizehint!(rowgroups, length(plan.rowgroups)) + indexgroups = nothing + if pageindex + _reservearray!(budget, Vector{Metadata.OffsetIndex}, + length(plan.rowgroups)) + indexgroups = Vector{Metadata.OffsetIndex}[] + sizehint!(indexgroups, length(plan.rowgroups)) + end + emitordinals = length(plan.rowgroups) <= Int(typemax(Int16)) + 1 + for (groupindex, rowgroupplan) in enumerate(plan.rowgroups) + _validatewritechoicecount(rowgroupplan.leaves, choices) + _reservearray!(budget, Metadata.ColumnChunk, + length(rowgroupplan.leaves)) + chunks = Metadata.ColumnChunk[] + sizehint!(chunks, length(rowgroupplan.leaves)) + indexes = nothing + if pageindex + _reservearray!(budget, Metadata.OffsetIndex, + length(rowgroupplan.leaves)) + indexes = Metadata.OffsetIndex[] + sizehint!(indexes, length(rowgroupplan.leaves)) + end + uncompressedsize = Int64(0) + compressedsize = Int64(0) + groupoffset = Int64(length(output)) + for (leaf, choice) in zip(rowgroupplan.leaves, choices) + column = leaf.column + columnstatistics = if statistics + element = plan.schema.leaves[Int(leaf.ordinal)].element + _writecolumnstatistics(leaf, element, statisticslimit, budget) + else + nothing + end + offset = Int64(length(output)) + pages, pagecharge = _budgetedsplitcolumnpages(leaf, limits, budget; + pagesize=target, checksum=checksum, + dictionary=choice.dictionary, codec=compression, + compressionlevel=compressionlevel, pageversion=version, + encoding=choice.encoding, capturelocations=pageindex) + _reserveoutputgrowth!(budget, length(pages.bytes)) + append!(output, pages.bytes) + _reserveobjects!(budget, 4) + push!(chunks, _columnchunk(leaf, offset, pages, compression, + columnstatistics)) + pageindex && push!(indexes, _writeabsoluteoffsetindex(pages, + offset, rowgroupplan.rows, budget)) + transferred = _columnpagestransferredbytes(pages) + transferred <= pagecharge || throw(AssertionError( + "writer page metadata exceeds its live-page charge")) + uncompressedsize = _addgroupsize(uncompressedsize, pages.uncompressed_size) + compressedsize = _addgroupsize(compressedsize, Int64(length(pages.bytes))) + _release!(budget, pagecharge - transferred) + end + _reserveobjects!(budget, 2) + push!(rowgroups, Metadata.RowGroup( + columns=chunks, + total_byte_size=uncompressedsize, + num_rows=Int64(rowgroupplan.rows), + total_compressed_size=compressedsize, + file_offset=groupoffset, + ordinal=emitordinals ? Int16(groupindex - 1) : nothing, + )) + pageindex && push!(indexgroups, indexes) + end + if pageindex + obsolete = _writeoffsetindexobsoletebytes(rowgroups, indexgroups) + rowgroups = _writeoffsetindexsection!(output, rowgroups, + indexgroups, limits, budget) + indexgroups = nothing + indexes = nothing + chunks = nothing + _release!(budget, obsolete) + end + _reserveobjects!(budget, 2) + metadata = Metadata.FileMetaData( + version=Int32(1), + schema=plan.elements, + num_rows=Int64(plan.rows), + row_groups=rowgroups, + key_value_metadata=_writekeyvaluemetadata(table, limits, budget), + created_by="Parquet.jl version 1.0.0-DEV", + column_orders=columnorders, + ) + footer, footercharge = _writeencodefooter(metadata, limits, budget) + try + _reserveoutputgrowth!(budget, _materializedsum(length(footer), 8)) + append!(output, footer) + _writelittle!(output, UInt32(length(footer))) + append!(output, PARQUET_MAGIC) + finally + _release!(budget, footercharge) + end + return output +end + +""" + Parquet.write(sink, table; encoding=nothing, dictionary=false, + rowgroupsize=1_048_576, pagesize=1024 * 1024, pageindex=true, + statistics=true, + kwargs...) + +Write a Tables.jl source to a path or `IO`. An encoding Symbol or string applies to +every column. A `Pair`, `NamedTuple`, or dictionary overrides exact column names. +Unlisted columns use PLAIN, or adaptive dictionary encoding when `dictionary=true`. +Use `:dictionary` to request adaptive dictionary encoding for one selected column. +Footer statistics and a complete column-order declaration are emitted by default. +Set `statistics=false` to omit them. `limits.max_statistics_value_bytes` limits +each emitted bound. +""" +function write(io::IO, table; checksum::Bool=true, dictionary::Bool=false, + encoding=nothing, codec=:uncompressed, + compressionlevel::Union{Nothing,Integer}=nothing, pageversion=:v1, + rowgroupsize=1_048_576, pagesize=1024 * 1024, pageindex::Bool=true, + statistics::Bool=true, limits::Limits=Limits()) + _statisticlimit(limits) + bytes = _encodefile(table; checksum=checksum, dictionary=dictionary, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion, + encoding=encoding, rowgroupsize=rowgroupsize, pagesize=pagesize, + pageindex=pageindex, statistics=statistics, limits=limits) + Base.write(io, bytes) + return +end + +function write(path::AbstractString, table; checksum::Bool=true, dictionary::Bool=false, + encoding=nothing, codec=:uncompressed, compressionlevel::Union{Nothing,Integer}=nothing, + pageversion=:v1, rowgroupsize=1_048_576, pagesize=1024 * 1024, + pageindex::Bool=true, statistics::Bool=true, limits::Limits=Limits()) + _statisticlimit(limits) + bytes = _encodefile(table; checksum=checksum, dictionary=dictionary, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion, + encoding=encoding, rowgroupsize=rowgroupsize, pagesize=pagesize, + pageindex=pageindex, statistics=statistics, limits=limits) + open(path, "w") do io + Base.write(io, bytes) + end + return +end diff --git a/src/write_logical.jl b/src/write_logical.jl new file mode 100644 index 0000000..9264dd8 --- /dev/null +++ b/src/write_logical.jl @@ -0,0 +1,353 @@ +function _canonicaltimeunit(unit::UInt8) + unit == _TEMPORAL_MILLIS && + return Metadata.TimeUnit(MILLIS=Metadata.MilliSeconds()) + unit == _TEMPORAL_MICROS && + return Metadata.TimeUnit(MICROS=Metadata.MicroSeconds()) + unit == _TEMPORAL_NANOS && + return Metadata.TimeUnit(NANOS=Metadata.NanoSeconds()) + throw(ArgumentError("unknown temporal unit code $unit")) +end + +function _canonicalconverted(kind::_TimeLogicalKind) + kind.unit == _TEMPORAL_MILLIS && return Metadata.ConvertedType.TIME_MILLIS + kind.unit == _TEMPORAL_MICROS && return Metadata.ConvertedType.TIME_MICROS + return nothing +end + +function _canonicalconverted(kind::_TimestampLogicalKind) + kind.unit == _TEMPORAL_MILLIS && return Metadata.ConvertedType.TIMESTAMP_MILLIS + kind.unit == _TEMPORAL_MICROS && return Metadata.ConvertedType.TIMESTAMP_MICROS + return nothing +end + +function _canonicalconverted(kind::_IntegerLogicalKind) + if kind.signed + kind.bitwidth == 8 && return Metadata.ConvertedType.INT_8 + kind.bitwidth == 16 && return Metadata.ConvertedType.INT_16 + kind.bitwidth == 32 && return Metadata.ConvertedType.INT_32 + return Metadata.ConvertedType.INT_64 + end + kind.bitwidth == 8 && return Metadata.ConvertedType.UINT_8 + kind.bitwidth == 16 && return Metadata.ConvertedType.UINT_16 + kind.bitwidth == 32 && return Metadata.ConvertedType.UINT_32 + return Metadata.ConvertedType.UINT_64 +end + +function _canonicaltemporallogical(element::Metadata.SchemaElement, + kind::_TimeLogicalKind) + logical = element.logicalType + annotation = logical === nothing ? Metadata.TimeType( + isAdjustedToUTC=kind.is_adjusted_to_utc, + unit=_canonicaltimeunit(kind.unit)) : logical.TIME + return Metadata.LogicalType(TIME=annotation), _canonicalconverted(kind) +end + +function _canonicaltemporallogical(element::Metadata.SchemaElement, + kind::_TimestampLogicalKind) + logical = element.logicalType + annotation = logical === nothing ? Metadata.TimestampType( + isAdjustedToUTC=kind.is_adjusted_to_utc, + unit=_canonicaltimeunit(kind.unit)) : logical.TIMESTAMP + return Metadata.LogicalType(TIMESTAMP=annotation), _canonicalconverted(kind) +end + +function _canonicaltemporallogical(element::Metadata.SchemaElement, + kind::_IntegerLogicalKind) + logical = element.logicalType + annotation = logical === nothing ? Metadata.IntType( + bitWidth=Int8(kind.bitwidth), isSigned=kind.signed) : logical.INTEGER + return Metadata.LogicalType(INTEGER=annotation), _canonicalconverted(kind) +end + +function _canonicalbinarylogical(element::Metadata.SchemaElement, kind::Symbol) + logical = element.logicalType + if kind === :enum + annotation = logical === nothing ? Metadata.EnumType() : logical.ENUM + return Metadata.LogicalType(ENUM=annotation), Metadata.ConvertedType.ENUM + elseif kind === :json + annotation = logical === nothing ? Metadata.JsonType() : logical.JSON + return Metadata.LogicalType(JSON=annotation), Metadata.ConvertedType.JSON + elseif kind === :bson + annotation = logical === nothing ? Metadata.BsonType() : logical.BSON + return Metadata.LogicalType(BSON=annotation), Metadata.ConvertedType.BSON + elseif kind === :uuid + return Metadata.LogicalType(UUID=logical.UUID), nothing + elseif kind === :float16 + return Metadata.LogicalType(FLOAT16=logical.FLOAT16), nothing + elseif kind === :unknown + return Metadata.LogicalType(UNKNOWN=logical.UNKNOWN), nothing + elseif kind === :interval + return nothing, Metadata.ConvertedType.INTERVAL + end + throw(ArgumentError("unsupported binary logical kind $kind")) +end + +function _canonicalannotation(element::Metadata.SchemaElement, kind) + if kind === :string + annotation = element.logicalType === nothing ? Metadata.StringType() : + element.logicalType.STRING + return Metadata.LogicalType(STRING=annotation), Metadata.ConvertedType.UTF8, + nothing, nothing + elseif kind === :date + annotation = element.logicalType === nothing ? Metadata.DateType() : + element.logicalType.DATE + return Metadata.LogicalType(DATE=annotation), Metadata.ConvertedType.DATE, + nothing, nothing + elseif kind === :decimal + precision, scale = something(_decimalparameters(element)) + annotation = element.logicalType === nothing ? Metadata.DecimalType( + scale=scale, precision=precision) : element.logicalType.DECIMAL + return Metadata.LogicalType(DECIMAL=annotation), + Metadata.ConvertedType.DECIMAL, scale, precision + elseif kind isa Union{_TimeLogicalKind,_TimestampLogicalKind,_IntegerLogicalKind} + logical, converted = _canonicaltemporallogical(element, kind) + return logical, converted, nothing, nothing + end + logical, converted = _canonicalbinarylogical(element, kind) + return logical, converted, nothing, nothing +end + +function _canonicalwriteelement(element::Metadata.SchemaElement) + kind = _logicalkind(element) + kind === nothing && return element + logical, converted, scale, precision = _canonicalannotation(element, kind) + return Metadata.SchemaElement( + type_=element.type_, + type_length=element.type_length, + repetition_type=element.repetition_type, + name=element.name, + num_children=element.num_children, + converted_type=converted, + scale=scale, + precision=precision, + field_id=element.field_id, + logicalType=logical, + unknown_fields=element.unknown_fields, + ) +end + +function _writescalarlogicalcolumn(name, values::AbstractVector, + element::Metadata.SchemaElement, limits::Limits) + element = _canonicalwriteelement(element) + optional = Missing <: eltype(values) + physical = _physicalvalues(element, values; limits=limits) + definition = Int16(optional ? 1 : 0) + schema = Metadata.SchemaElement[element] + return WriteColumn(String(name), physical, element.type_, element.type_length, + optional, element.logicalType, element.converted_type, String[String(name)], + nothing, nothing, Int16(0), definition, length(values), schema) +end + +function _logicalwriteelement(name, physical::Metadata.Type.T, optional::Bool; + width=nothing, logical=nothing, converted=nothing, scale=nothing, + precision=nothing) + repetition = optional ? Metadata.FieldRepetitionType.OPTIONAL : + Metadata.FieldRepetitionType.REQUIRED + return Metadata.SchemaElement( + type_=physical, + type_length=width, + repetition_type=repetition, + name=String(name), + converted_type=converted, + scale=scale, + precision=precision, + logicalType=logical, + ) +end + +function _binarywriteelement(name, value_type::Type, optional::Bool) + if value_type == UUIDs.UUID + return _logicalwriteelement(name, Metadata.Type.FIXED_LEN_BYTE_ARRAY, optional; + width=Int32(16), logical=Metadata.LogicalType(UUID=Metadata.UUIDType())) + elseif value_type == Float16 + return _logicalwriteelement(name, Metadata.Type.FIXED_LEN_BYTE_ARRAY, optional; + width=Int32(2), logical=Metadata.LogicalType(FLOAT16=Metadata.Float16Type())) + elseif value_type == JSONValue + return _logicalwriteelement(name, Metadata.Type.BYTE_ARRAY, optional; + logical=Metadata.LogicalType(JSON=Metadata.JsonType()), + converted=Metadata.ConvertedType.JSON) + elseif value_type == BSONValue + return _logicalwriteelement(name, Metadata.Type.BYTE_ARRAY, optional; + logical=Metadata.LogicalType(BSON=Metadata.BsonType()), + converted=Metadata.ConvertedType.BSON) + elseif value_type == Interval + return _logicalwriteelement(name, Metadata.Type.FIXED_LEN_BYTE_ARRAY, optional; + width=Int32(12), converted=Metadata.ConvertedType.INTERVAL) + end + return nothing +end + +function _integerconverted(::Type{Int8}) + return Metadata.ConvertedType.INT_8 +end + +function _integerconverted(::Type{Int16}) + return Metadata.ConvertedType.INT_16 +end + +function _integerconverted(::Type{UInt8}) + return Metadata.ConvertedType.UINT_8 +end + +function _integerconverted(::Type{UInt16}) + return Metadata.ConvertedType.UINT_16 +end + +function _integerconverted(::Type{UInt32}) + return Metadata.ConvertedType.UINT_32 +end + +function _integerconverted(::Type{UInt64}) + return Metadata.ConvertedType.UINT_64 +end + +function _integerwriteelement(name, value_type::Type, optional::Bool) + value_type in (Int8, Int16, UInt8, UInt16, UInt32, UInt64) || return nothing + width = Int8(8 * sizeof(value_type)) + signed = value_type <: Signed + physical = sizeof(value_type) <= 4 ? Metadata.Type.INT32 : Metadata.Type.INT64 + logical = Metadata.LogicalType( + INTEGER=Metadata.IntType(bitWidth=width, isSigned=signed)) + return _logicalwriteelement(name, physical, optional; logical=logical, + converted=_integerconverted(value_type)) +end + +function _timestampwriteadjustment(values::AbstractVector) + adjusted = nothing + for value in values + ismissing(value) && continue + value isa Timestamp || throw(ArgumentError( + "TIMESTAMP columns must contain Timestamp values or missing")) + if adjusted === nothing + adjusted = value.is_adjusted_to_utc + else + value.is_adjusted_to_utc == adjusted || throw(ArgumentError( + "all values in a TIMESTAMP column must use the same UTC adjustment")) + end + end + adjusted === nothing && throw(ArgumentError( + "cannot infer TIMESTAMP UTC adjustment from an empty or all-null column")) + return adjusted +end + +function _timestampwriteunitcode(value_type::Type) + value_type == Timestamp{:millis} && return _TEMPORAL_MILLIS + value_type == Timestamp{:micros} && return _TEMPORAL_MICROS + value_type == Timestamp{:nanos} && return _TEMPORAL_NANOS + throw(ArgumentError("unsupported TIMESTAMP element type $value_type")) +end + +function _timestampwriteunit(value_type::Type) + return _canonicaltimeunit(_timestampwriteunitcode(value_type)) +end + +function _temporalwriteelement(name, values::AbstractVector, value_type::Type, + optional::Bool) + integer = _integerwriteelement(name, value_type, optional) + integer === nothing || return integer + if value_type == Dates.Time + unit = Metadata.TimeUnit(NANOS=Metadata.NanoSeconds()) + logical = Metadata.LogicalType( + TIME=Metadata.TimeType(isAdjustedToUTC=false, unit=unit)) + return _logicalwriteelement(name, Metadata.Type.INT64, optional; + logical=logical) + elseif value_type == Dates.DateTime + unit = Metadata.TimeUnit(MILLIS=Metadata.MilliSeconds()) + logical = Metadata.LogicalType( + TIMESTAMP=Metadata.TimestampType(isAdjustedToUTC=false, unit=unit)) + return _logicalwriteelement(name, Metadata.Type.INT64, optional; + logical=logical, + converted=Metadata.ConvertedType.TIMESTAMP_MILLIS) + elseif value_type <: Timestamp + adjusted = _timestampwriteadjustment(values) + unit = _timestampwriteunit(value_type) + logical = Metadata.LogicalType( + TIMESTAMP=Metadata.TimestampType(isAdjustedToUTC=adjusted, unit=unit)) + return _logicalwriteelement(name, Metadata.Type.INT64, optional; + logical=logical, + converted=_canonicalconverted( + _TimestampLogicalKind(_timestampwriteunitcode(value_type), adjusted))) + end + return nothing +end + +function _decimalwriteparameters(values::AbstractVector) + scale = nothing + precision = 1 + for value in values + ismissing(value) && continue + value isa Decimal || throw(ArgumentError( + "DECIMAL columns must contain Decimal values or missing")) + if scale === nothing + scale = value.scale + else + value.scale == scale || throw(ArgumentError( + "all values in a DECIMAL column must have the same scale")) + end + precision = max(precision, _decimaldigits(value.unscaled)) + end + scale === nothing && throw(ArgumentError( + "cannot infer DECIMAL scale from an empty or all-null column")) + precision = max(precision, Int(scale)) + precision <= typemax(Int32) || throw(ArgumentError( + "DECIMAL precision exceeds Int32")) + return Int32(precision), scale +end + +function _decimalwritewidth(precision::Int32, limits::Limits) + width = setprecision(BigFloat, 256) do + bits = ceil(Int64, BigFloat(precision) * log2(BigFloat(10)) + 1) + return max(Int64(1), cld(bits, Int64(8))) + end + while _fixeddecimalprecision(width) < precision + width = Base.checked_add(width, Int64(1)) + end + while width > 1 && _fixeddecimalprecision(width - 1) >= precision + width -= 1 + end + _checklimit(:string_bytes, width, limits.max_string_bytes) + width <= typemax(Int32) || throw(ArgumentError( + "DECIMAL fixed width exceeds Int32")) + return Int32(width) +end + +function _decimalwriteelement(name, values::AbstractVector, optional::Bool, + limits::Limits) + precision, scale = _decimalwriteparameters(values) + if precision <= 9 + physical = Metadata.Type.INT32 + width = nothing + elseif precision <= 18 + physical = Metadata.Type.INT64 + width = nothing + else + physical = Metadata.Type.FIXED_LEN_BYTE_ARRAY + width = _decimalwritewidth(precision, limits) + end + logical = Metadata.LogicalType( + DECIMAL=Metadata.DecimalType(scale=scale, precision=precision)) + return _logicalwriteelement(name, physical, optional; width=width, + logical=logical, converted=Metadata.ConvertedType.DECIMAL, + scale=scale, precision=precision) +end + +function _logicalwritecolumn(name, values::AbstractVector, value_type::Type, + limits::Limits) + optional = Missing <: eltype(values) + element = _temporalwriteelement(name, values, value_type, optional) + element = if element !== nothing + element + elseif value_type == Decimal + _decimalwriteelement(name, values, optional, limits) + else + _binarywriteelement(name, value_type, optional) + end + element === nothing && return nothing + return _writescalarlogicalcolumn(name, values, element, limits) +end + +function _unknowncolumn(name, values::AbstractVector, limits::Limits) + element = _logicalwriteelement(name, Metadata.Type.INT32, true; + logical=Metadata.LogicalType(UNKNOWN=Metadata.NullType())) + return _writescalarlogicalcolumn(name, values, element, limits) +end diff --git a/src/write_nested.jl b/src/write_nested.jl new file mode 100644 index 0000000..7f5edd2 --- /dev/null +++ b/src/write_nested.jl @@ -0,0 +1,6780 @@ +# Iterative declared-type inference and Dremel shredding for canonical nested writes. + +abstract type _NestedWriteShape end + +mutable struct _NestedWriteLeafAggregate + seen::Bool + adjusted::Union{Nothing,Bool} + scale::Union{Nothing,Int32} + precision::Int32 +end + +struct _NestedWriteLeafShape <: _NestedWriteShape + name::String + value_type::Type + optional::Bool + explicit::Union{Nothing,_ScalarLogicalColumnSpec} + fixed_width::Union{Nothing,Int32} + aggregate::_NestedWriteLeafAggregate + source_snapshot::Any +end + + +function _NestedWriteLeafShape(name::String, value_type::Type, optional::Bool, + explicit::Union{Nothing,_ScalarLogicalColumnSpec}, + fixed_width::Union{Nothing,Int32}, + aggregate::_NestedWriteLeafAggregate) + return _NestedWriteLeafShape(name, value_type, optional, explicit, + fixed_width, aggregate, nothing) +end + +struct _NestedWriteStructShape <: _NestedWriteShape + name::String + value_type::Type + optional::Bool + names::Vector{String} + children::Vector{_NestedWriteShape} + source_children::Union{Nothing,Vector{AbstractVector}} + source_snapshot::Any + package_owned::Bool +end + +struct _NestedWriteListShape <: _NestedWriteShape + name::String + value_type::Type + optional::Bool + element::_NestedWriteShape + source_snapshot::Any +end + +struct _NestedWriteMapShape <: _NestedWriteShape + name::String + value_type::Type + optional::Bool + key::_NestedWriteShape + value::_NestedWriteShape + source_snapshot::Any + package_owned::Bool + source_has_values::Bool +end + +abstract type _NestedWriteNodePlan end + +struct _NestedWriteLeafPlan <: _NestedWriteNodePlan + semantic::_NestedLeafPlan + shape::_NestedWriteLeafShape +end + +struct _NestedWriteStructPlan <: _NestedWriteNodePlan + semantic::_NestedStructPlan + shape::_NestedWriteStructShape + children::Vector{_NestedWriteNodePlan} +end + +struct _NestedWriteListPlan <: _NestedWriteNodePlan + semantic::_NestedListPlan + shape::_NestedWriteListShape + element::_NestedWriteNodePlan +end + +struct _NestedWriteMapPlan <: _NestedWriteNodePlan + semantic::_NestedMapPlan + shape::_NestedWriteMapShape + key::_NestedWriteNodePlan + value::_NestedWriteNodePlan +end + +mutable struct _NestedWriteLeafCount + entries::Int64 + dense::Int64 + payload_bytes::Int64 +end + +mutable struct _NestedWriteLeafBuilder + repetition::Vector{UInt64} + definition::Vector{UInt64} + values::Vector + entry_position::Int + dense_position::Int + payload_position::Int64 +end + +const _NESTED_WRITE_TRACE_LEAF = UInt8(1) +const _NESTED_WRITE_TRACE_STRUCT = UInt8(2) +const _NESTED_WRITE_TRACE_LIST = UInt8(3) +const _NESTED_WRITE_TRACE_MAP = UInt8(4) +const _NESTED_WRITE_TRACE_KEY = UInt8(5) +const _NESTED_WRITE_TRACE_SOURCE = UInt8(6) +const _NESTED_WRITE_TRACE_CHUNK = 128 + +const _NESTED_WRITE_KEY_SCALAR = UInt8(1) +const _NESTED_WRITE_KEY_BYTES = UInt8(2) +const _NESTED_WRITE_KEY_STRING = UInt8(3) +const _NESTED_WRITE_KEY_TUPLE = UInt8(4) +const _NESTED_WRITE_KEY_NAMED_TUPLE = UInt8(5) +const _NESTED_WRITE_KEY_LIST = UInt8(6) +const _NESTED_WRITE_KEY_STRUCT = UInt8(7) +const _NESTED_WRITE_KEY_MAP = UInt8(8) +const _NESTED_WRITE_KEY_PAIR = UInt8(9) + +struct _NestedWriteKeySnapshot + kind::UInt8 + source_type::Type + first::Int + last::Int + value::Any + names::Any + children::Vector{_NestedWriteKeySnapshot} +end + +const _NESTED_WRITE_EMPTY_KEYS = _NestedWriteKeySnapshot[] + +struct _NestedWriteKeyCollectionState + count::Int + first::Int + last::Int + indexed::Bool + source::Any +end + +mutable struct _NestedWriteKeyCaptureFrame + parent::Union{Nothing,_NestedWriteKeyCaptureFrame} + parent_slot::Int + value::Any + shape::Union{Nothing,_NestedWriteShape} + witness::Any + row_witness::Any + state::_NestedWriteKeyCollectionState + depth::Int + kind::UInt8 + children::Vector{_NestedWriteKeySnapshot} + names::Any + position::Int + iterator::Any + iteration::Any + pending::Any + next_value::Any + next_shape::Union{Nothing,_NestedWriteShape} + next_witness::Any + next_row_witness::Any +end + +mutable struct _NestedWriteKeyCompareFrame + parent::Union{Nothing,_NestedWriteKeyCompareFrame} + expected::_NestedWriteKeySnapshot + current::Any + shape::Union{Nothing,_NestedWriteShape} + witness::Any + row_witness::Any + state::_NestedWriteKeyCollectionState + depth::Int + position::Int + iterator::Any + iteration::Any + pending::Any + next_expected::Union{Nothing,_NestedWriteKeySnapshot} + next_current::Any + next_shape::Union{Nothing,_NestedWriteShape} + next_witness::Any + next_row_witness::Any +end + +struct _NestedWriteTraceEvent + kind::UInt8 + a::Int64 + b::Int64 + c::Int64 + d::Int64 + value::Any +end + +mutable struct _NestedWriteTraceChunk + events::Vector{_NestedWriteTraceEvent} + used::Int + next::Union{Nothing,_NestedWriteTraceChunk} +end + +mutable struct _NestedWriteTrace + budget::_LiveByteBudget + first::Union{Nothing,_NestedWriteTraceChunk} + last::Union{Nothing,_NestedWriteTraceChunk} + current::Union{Nothing,_NestedWriteTraceChunk} + position::Int + capturing::Bool + charge::Int64 + topology::Any +end + +const _NESTED_WRITE_VECTOR_GENERIC = UInt8(0) +const _NESTED_WRITE_VECTOR_LOGICAL = UInt8(1) +const _NESTED_WRITE_VECTOR_FIXED = UInt8(2) +const _NESTED_WRITE_VECTOR_LIST = UInt8(3) +const _NESTED_WRITE_VECTOR_STRUCT = UInt8(4) +const _NESTED_WRITE_VECTOR_MAP = UInt8(5) + +struct _NestedWriteVectorSnapshot + source::AbstractVector + kind::UInt8 + count::Int + first::Int + last::Int + primary::Any + secondary::Any + tertiary::Any + quaternary::Any + copy1::Any + copy2::Any + copy3::Any + scalar::Int64 +end + +mutable struct _NestedWriteSnapshotLookup + sources::Vector{Union{Nothing,AbstractVector}} + snapshots::Vector{Union{Nothing,_NestedWriteVectorSnapshot}} + count::Int + charge::Int64 +end + +mutable struct _NestedWriteDictEntry + key::Any + value::Any + next::Any +end + +mutable struct _NestedWriteDictDependency + snapshot::Any + next::Any +end + +struct _NestedWriteStructViewSnapshot + source::StructValue + names::Vector{String} + children::Vector{AbstractVector} + names_copy::Vector{String} + children_copy::Vector{AbstractVector} + index::Int +end + +mutable struct _NestedWriteIdentitySet + sources::Vector{Union{Nothing,AbstractVector}} + count::Int +end + +mutable struct _NestedWriteDictWork + value::Any + shape::Union{Nothing,_NestedWriteShape} + depth::Int + source::Bool + next::Any +end + +mutable struct _NestedWriteDictMaterialization + first::Any + last::Any + dependencies::Any + dependency_last::Any + dependency_sources::Any + preflight_sources::Any + count::Int + charge::Int64 +end + +mutable struct _NestedWriteTopologySnapshot + input_columns::Union{Nothing,AbstractVector} + input_count::Int + input_first::Int + input_last::Int + names::Vector{String} + values::Vector{AbstractVector} + nodes::Vector{_NestedWriteVectorSnapshot} + lookup::_NestedWriteSnapshotLookup + base_nodes::Int + charge::Int64 +end + +struct _NestedWriteRowWitness + snapshot::_NestedWriteVectorSnapshot + index::Int + first::Int + last::Int + present::Bool +end + +struct _NestedWriteSourceFrame + values::AbstractVector + depth::Int + position::Int + count::Int + exit::Bool +end + +const _NESTED_WRITE_SHAPE_STRUCT = UInt8(1) +const _NESTED_WRITE_SHAPE_LIST = UInt8(2) +const _NESTED_WRITE_SHAPE_MAP = UInt8(3) + +struct _NestedWriteShapeFrame + kind::UInt8 + name::String + value_type::Type + optional::Bool + source::Any + depth::Int + names::Union{Nothing,Vector{String}} + source_children::Union{Nothing,Vector{AbstractVector}} + child_types::Any + first_source::Any + second_source::Any + children::Union{Nothing,Vector{_NestedWriteShape}} + first::Union{Nothing,_NestedWriteShape} + second::Union{Nothing,_NestedWriteShape} + position::Int + package_owned::Bool + source_has_values::Bool +end + +struct _NestedWriteSchemaFrame + shape::_NestedWriteShape + fragments::Union{Nothing,Vector{Vector{Metadata.SchemaElement}}} + first::Union{Nothing,Vector{Metadata.SchemaElement}} + second::Union{Nothing,Vector{Metadata.SchemaElement}} + position::Int + count::Int +end + +struct _NestedWriteBindFrame + shape::_NestedWriteShape + semantic::_NestedPlan + children::Union{Nothing,Vector{_NestedWriteNodePlan}} + first::Union{Nothing,_NestedWriteNodePlan} + second::Union{Nothing,_NestedWriteNodePlan} + position::Int +end + +const _NESTED_WRITE_SCAN_ENTER = UInt8(1) +const _NESTED_WRITE_SCAN_POSTCHECK = UInt8(2) +const _NESTED_WRITE_SCAN_STRUCT = UInt8(3) +const _NESTED_WRITE_SCAN_LIST = UInt8(4) +const _NESTED_WRITE_SCAN_MAP_DICT = UInt8(5) +const _NESTED_WRITE_SCAN_MAP_VIEW_KEY = UInt8(6) +const _NESTED_WRITE_SCAN_MAP_VIEW_VALUE = UInt8(7) +const _NESTED_WRITE_SCAN_KEYASSERT = UInt8(11) +const _NESTED_WRITE_SCAN_DICT_RELEASE = UInt8(12) + +struct _NestedWriteScanAction + kind::UInt8 + shape::_NestedWriteShape + value::Any + row_witness::Any + other_witness::Any + state::Any + expected::Union{Nothing,_NestedWriteKeySnapshot} + materialization::Union{Nothing,_NestedWriteDictMaterialization} + snapshot1::Any + snapshot2::Any + position::Int + count::Int + first::Int + last::Int +end + +const _NESTED_WRITE_SHRED_ENTER = UInt8(1) +const _NESTED_WRITE_SHRED_POSTCHECK = UInt8(2) +const _NESTED_WRITE_SHRED_STRUCT = UInt8(3) +const _NESTED_WRITE_SHRED_LIST = UInt8(4) +const _NESTED_WRITE_SHRED_MAP_DICT = UInt8(5) +const _NESTED_WRITE_SHRED_MAP_VIEW_KEY = UInt8(6) +const _NESTED_WRITE_SHRED_MAP_VIEW_VALUE = UInt8(7) +const _NESTED_WRITE_SHRED_KEYASSERT = UInt8(11) +const _NESTED_WRITE_SHRED_DICT_RELEASE = UInt8(12) + +struct _NestedWriteShredAction + kind::UInt8 + plan::_NestedWriteNodePlan + value::Any + expected::Union{Nothing,_NestedWriteKeySnapshot} + keymode::Bool + repetition::UInt64 + row_witness::Any + other_witness::Any + state::Any + nested::Union{Nothing,_NestedWriteKeySnapshot} + materialization::Union{Nothing,_NestedWriteDictMaterialization} + snapshot1::Any + snapshot2::Any + position::Int + count::Int + first::Int + last::Int + rawvalue::Any + value_witness::Any + phase::UInt8 +end + +mutable struct _NestedWritePassStack{T} + frames::Vector{T} + highwater::Int + processing::Bool + charge::Int64 +end + +function Base.isempty(stack::_NestedWritePassStack) + return isempty(stack.frames) +end + +function Base.lastindex(stack::_NestedWritePassStack) + return lastindex(stack.frames) +end + +function Base.getindex(stack::_NestedWritePassStack, index::Int) + return stack.frames[index] +end + +function Base.setindex!(stack::_NestedWritePassStack{T}, frame::T, + index::Int) where {T} + stack.frames[index] = frame + return frame +end + +function _NestedWriteShredAction(kind::UInt8, plan::_NestedWriteNodePlan, + value, expected::Union{Nothing,_NestedWriteKeySnapshot}, keymode::Bool, + repetition::UInt64, row_witness, other_witness, state, + nested::Union{Nothing,_NestedWriteKeySnapshot}, + materialization::Union{Nothing,_NestedWriteDictMaterialization}, + snapshot1, snapshot2, position::Int, count::Int, first::Int, last::Int, + phase::UInt8) + return _NestedWriteShredAction(kind, plan, value, expected, keymode, + repetition, row_witness, other_witness, state, nested, + materialization, snapshot1, snapshot2, position, count, first, last, + nothing, nothing, phase) +end + +function _nestedwritepassstackstart(::Type{T}, + budget::_LiveByteBudget) where {T} + charge = _materializedsum(_materializedarraybytes(T, 0), + _MATERIALIZED_OBJECT_BYTES) + _reserve!(budget, charge) + try + return _NestedWritePassStack(T[], 0, false, charge) + catch + _release!(budget, charge) + rethrow() + end +end + +function _nestedwritestackpush!(stack::_NestedWritePassStack{T}, frame::T, + budget::_LiveByteBudget) where {T} + iszero(stack.charge) && throw(AssertionError( + "nested writer cannot reuse a released pass stack")) + active = length(stack.frames) + 1 + (stack.processing ? 1 : 0) + added = active > stack.highwater + framecharge = added ? _materializedarraybytes(T, 1; header=false) : + Int64(0) + added && _reserve!(budget, framecharge) + try + push!(stack.frames, frame) + catch + added && _release!(budget, framecharge) + rethrow() + end + if added + stack.highwater = active + stack.charge = _materializedsum(stack.charge, framecharge) + end + return +end + +function _nestedwritepassstackpop!(stack::_NestedWritePassStack) + frame = pop!(stack.frames) + stack.processing = true + return frame +end + +function _nestedwritepassstackprocessed!(stack::_NestedWritePassStack) + stack.processing || throw(AssertionError( + "nested writer pass stack has no active action")) + stack.processing = false + return +end + +function _nestedwritepassstackclear!(stack::_NestedWritePassStack) + empty!(stack.frames) + stack.processing = false + return +end + +function _nestedwritepassstackrelease!(stack::_NestedWritePassStack, + budget::_LiveByteBudget) + _nestedwritepassstackclear!(stack) + charge = stack.charge + stack.highwater = 0 + stack.charge = Int64(0) + _release!(budget, charge) + return +end + +function _nestedwritestackpop!(stack::_NestedWritePassStack, + ::_LiveByteBudget) + return pop!(stack.frames) +end + +function _nestedwritedepthadd(depth::Int, increment::Int, limits::Limits) + requested = try + Base.checked_add(depth, increment) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:metadata_depth, typemax(Int64), + limits.max_metadata_depth)) + end + return requested +end + +function _nestedwritesourcechildcount(values::AbstractVector) + Base.@nospecialize values + values isa Union{LogicalColumn,FixedByteArrayVector,ListVector} && return 1 + values isa StructVector && return length(values.children) + values isa MapVector && return values.values === nothing ? 1 : 2 + return 0 +end + +function _nestedwritesourcechild(values::AbstractVector, position::Int, + count::Int) + Base.@nospecialize values + _nestedwritesourcechildcount(values) == count || throw(ArgumentError( + "nested Parquet vector child count changed during traversal")) + 1 <= position <= count || throw(AssertionError( + "nested Parquet source traversal has an invalid child position")) + if values isa Union{LogicalColumn,FixedByteArrayVector,ListVector} + return values.values + elseif values isa StructVector + return values.children[position] + end + map = values::MapVector + position == 1 && return map.keys + return something(map.values) +end + +function _nestedwritesourceschedule!( + stack::_NestedWritePassStack{_NestedWriteSourceFrame}, + frame::_NestedWriteSourceFrame, limits::Limits, + budget::_LiveByteBudget; postorder::Bool=false) + position = frame.position + 1 + position > frame.count && return false + child = _nestedwritesourcechild(frame.values, position, frame.count) + childdepth = _nestedwritedepthadd(frame.depth, 1, limits) + _checklimit(:metadata_depth, childdepth, limits.max_metadata_depth) + if position < frame.count + _nestedwritestackpush!(stack, _NestedWriteSourceFrame(frame.values, + frame.depth, position, frame.count, false), budget) + elseif postorder + _nestedwritestackpush!(stack, _NestedWriteSourceFrame(frame.values, + frame.depth, frame.count, frame.count, true), budget) + end + _nestedwritestackpush!(stack, _NestedWriteSourceFrame(child, childdepth, + 0, -1, false), budget) + return true +end + +function _nestedwritetrace(budget::_LiveByteBudget) + return _nestedwritetrace(budget, nothing) +end + +function _nestedwritetrace(budget::_LiveByteBudget, topology) + charge = _reserveobjects!(budget) + return _NestedWriteTrace(budget, nothing, nothing, nothing, 1, true, + charge, topology) +end + +function _nestedwritetracereserve!(trace::_NestedWriteTrace, bytes::Int64) + _reserve!(trace.budget, bytes) + trace.charge = _materializedsum(trace.charge, bytes) + return +end + +function _nestedwritetraceunreserve!(trace::_NestedWriteTrace, bytes::Int64) + _release!(trace.budget, bytes) + trace.charge = Base.checked_sub(trace.charge, bytes) + return +end + +function _nestedwritetracechunk!(trace::_NestedWriteTrace) + charge = _materializedsum( + _materializedarraybytes(_NestedWriteTraceEvent, + _NESTED_WRITE_TRACE_CHUNK), _MATERIALIZED_OBJECT_BYTES) + _nestedwritetracereserve!(trace, charge) + events = Vector{_NestedWriteTraceEvent}(undef, + _NESTED_WRITE_TRACE_CHUNK) + chunk = _NestedWriteTraceChunk(events, 0, nothing) + if trace.last === nothing + trace.first = chunk + else + something(trace.last).next = chunk + end + trace.last = chunk + return chunk +end + +function _nestedwritetracecapture!(trace::_NestedWriteTrace, + event::_NestedWriteTraceEvent) + chunk = trace.last + if chunk === nothing || chunk.used == length(chunk.events) + chunk = _nestedwritetracechunk!(trace) + end + chunk.used += 1 + chunk.events[chunk.used] = event + return +end + +function _nestedwritetracenext!(trace::_NestedWriteTrace) + chunk = trace.current + chunk === nothing && throw(ArgumentError( + "nested Parquet input added occurrences between writer passes")) + position = trace.position + position <= chunk.used || throw(AssertionError( + "nested writer trace cursor exceeded its chunk")) + event = chunk.events[position] + position += 1 + if position > chunk.used + trace.current = chunk.next + trace.position = 1 + else + trace.position = position + end + return event +end + +function _nestedwritetracepeekkey(trace::_NestedWriteTrace) + chunk = trace.current + position = trace.position + while chunk !== nothing + while position <= chunk.used + event = chunk.events[position] + event.kind == _NESTED_WRITE_TRACE_KEY && return event + event.kind == _NESTED_WRITE_TRACE_SOURCE || throw(ArgumentError( + "nested Parquet input changed its MAP-key trace topology")) + position += 1 + end + chunk = chunk.next + position = 1 + end + throw(ArgumentError( + "nested Parquet input removed a MAP-key occurrence between writer passes")) +end + +function _nestedwritetraceevent!(trace::Nothing, ::UInt8, ::Int64, ::Int64, + ::Int64, ::Int64, value=nothing) + return +end + +function _nestedwritetraceevent!(trace::_NestedWriteTrace, kind::UInt8, + a::Int64, b::Int64, c::Int64, d::Int64, value=nothing) + if trace.capturing + _nestedwritetracecapture!(trace, + _NestedWriteTraceEvent(kind, a, b, c, d, value)) + return + end + expected = _nestedwritetracenext!(trace) + expected.kind == kind && expected.a == a && expected.b == b && + expected.c == c && expected.d == d || throw(ArgumentError( + "nested Parquet input changed its occurrence topology between writer passes")) + kind in (_NESTED_WRITE_TRACE_KEY, _NESTED_WRITE_TRACE_SOURCE) && throw( + AssertionError( + "nested Parquet identity events require bounded comparison")) + isequal(expected.value, value) || throw(ArgumentError( + "nested Parquet input changed its occurrence metadata between writer passes")) + return +end + +function _nestedwritetracecompare!(trace::_NestedWriteTrace) + trace.capturing = false + trace.current = trace.first + trace.position = 1 + return +end + +function _nestedwritetracefinishcompare!(trace::_NestedWriteTrace) + trace.current === nothing || throw(ArgumentError( + "nested Parquet input removed occurrences between writer passes")) + return +end + +function _nestedwritetracerelease!(trace::_NestedWriteTrace) + charge = trace.charge + iszero(charge) || _release!(trace.budget, charge) + trace.charge = Int64(0) + trace.first = nothing + trace.last = nothing + trace.current = nothing + return +end + +function _nestedwritekeynextdepth(depth::Int, limits::Limits) + depth < typemax(Int) || throw(LimitError(:metadata_depth, + typemax(Int64), limits.max_metadata_depth)) + return depth + 1 +end + +function _nestedwritekeychilddepth(count::Int, depth::Int, limits::Limits) + iszero(count) && return depth + nextdepth = _nestedwritekeynextdepth(depth, limits) + _checklimit(:metadata_depth, nextdepth, limits.max_metadata_depth) + return nextdepth +end + +function _nestedwritekeyrecursive(value) + value isa Union{Pair,NamedTuple,StructValue,MapValue,AbstractDict} && + return true + value isa Tuple && return !_nestedwriteisfixedtuple(typeof(value)) + value isa AbstractVector && !(value isa AbstractVector{UInt8}) && return true + return false +end + +function _nestedwritekeycaptureactive(value, + frame::Union{Nothing,_NestedWriteKeyCaptureFrame}) + current = frame + while current !== nothing + current.value === value && return true + current = current.parent + end + return false +end + +function _nestedwritekeycompareactive(value, + frame::Union{Nothing,_NestedWriteKeyCompareFrame}) + current = frame + while current !== nothing + current.current === value && return true + current = current.parent + end + return false +end + +function _nestedwritekeyframecharge!(trace::_NestedWriteTrace) + _nestedwritetracereserve!(trace, _MATERIALIZED_OBJECT_BYTES) + return +end + +function _nestedwritekeyframefree!(trace::_NestedWriteTrace) + _nestedwritetraceunreserve!(trace, _MATERIALIZED_OBJECT_BYTES) + return +end + +function _nestedwritekeycollectionfree!(trace::_NestedWriteTrace, + state::_NestedWriteKeyCollectionState) + source = state.source + source isa _NestedWriteDictMaterialization && + _nestedwritedictrelease!(trace, source) + return +end + +function _nestedwritekeycaptureframefree!(trace::_NestedWriteTrace, + frame::_NestedWriteKeyCaptureFrame) + _nestedwritekeycollectionfree!(trace, frame.state) + _nestedwritekeyframefree!(trace) + return +end + +function _nestedwritekeycompareframefree!(trace::_NestedWriteTrace, + frame::_NestedWriteKeyCompareFrame) + _nestedwritekeycollectionfree!(trace, frame.state) + _nestedwritekeyframefree!(trace) + return +end + +function _nestedwritekeydeclared(shape::Nothing, value) + return +end + +function _nestedwritekeydeclared(shape::_NestedWriteShape, value) + if ismissing(value) + getfield(shape, :optional) && return + elseif value isa getfield(shape, :value_type) + return + end + throw(ArgumentError("Parquet MAP key field $(repr(getfield(shape, :name))) " * + "contains $(typeof(value)); expected $(getfield(shape, :value_type))")) +end + +function _nestedwritekeywitnesssource(::Nothing, ::Int) + return nothing +end + +function _nestedwritekeywitnesschild(::Nothing, ::Int) + return nothing +end + +function _nestedwritekeywitnesslist(::Nothing) + return nothing +end + +function _nestedwritekeywitnessmapkey(::Nothing) + return nothing +end + +function _nestedwritekeywitnessmapvalue(::Nothing) + return nothing +end + +function _nestedwritekeywitnessroot(::Nothing, value) + return +end + +function _nestedwritekeywitnesssnapshot(::Nothing) + return nothing +end + +function _nestedwritekeyrowsnapshot(shape, witness, + trace::_NestedWriteTrace, source::AbstractVector, limits::Limits) + stored = _nestedwritekeywitnesssnapshot(witness) + if stored === nothing && shape !== nothing + stored = _nestedwritesourcesnapshot(shape) + end + return _nestedwritetraceviewsnapshot!(trace, source, stored, limits) +end + +function _nestedwritesourcesnapshot(shape::_NestedWriteLeafShape) + return shape.source_snapshot +end + +function _nestedwritesourcesnapshot(shape::_NestedWriteStructShape) + return shape.source_snapshot +end + +function _nestedwritesourcesnapshot(shape::_NestedWriteListShape) + return shape.source_snapshot +end + +function _nestedwritesourcesnapshot(shape::_NestedWriteMapShape) + return shape.source_snapshot +end + +function _nestedwriteoccurrencesnapshot!(trace, source::AbstractVector, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits) + stored = shape === nothing ? nothing : _nestedwritesourcesnapshot(shape) + if stored !== nothing && stored.source !== source + stored = nothing + end + return _nestedwritetraceviewsnapshot!(trace, source, stored, limits) +end + +function _nestedwriteispackagevector(value) + return value isa Union{LogicalColumn,FixedByteArrayVector,ListVector, + StructVector,MapVector} +end + +function _nestedwritekeystoredsource(::Nothing, ::Int) + return nothing +end + +function _nestedwritekeystoredsource(sources::Vector{AbstractVector}, + index::Int) + index <= length(sources) || return nothing + return sources[index] +end + +function _nestedwritekeycount(value) + if value isa AbstractVector + return _nestedvectorcount(value, "nested Parquet MAP key") + end + raw = length(value) + raw isa Int || throw(ArgumentError( + "nested Parquet MAP key length is not an Int")) + raw >= 0 || throw(ArgumentError( + "nested Parquet MAP key length must be nonnegative")) + return raw +end + +function _nestedwritekeyindexedstate(value) + count = _nestedwritekeycount(value) + first, last = value isa AbstractVector ? + _nestedvectoraxes(value, "nested Parquet MAP key") : + (_nestedvectoraxisvalue(firstindex(value), + "nested Parquet MAP key first index"), + _nestedvectoraxisvalue(lastindex(value), + "nested Parquet MAP key last index")) + return _NestedWriteKeyCollectionState(count, first, last, true, value) +end + +function _nestedwritekeycollectionstate(value) + return nothing +end + +function _nestedwritekeycollectionstate(value::Union{Pair,NamedTuple, + StructValue}) + return _NestedWriteKeyCollectionState( + _nestedwritekeycount(value), 0, 0, false, nothing) +end + +function _nestedwritekeycollectionstate(value::Tuple) + return _nestedwritekeyindexedstate(value) +end + +function _nestedwritekeycollectionstate(value::AbstractVector) + return _nestedwritekeyindexedstate(value) +end + +function _nestedwritekeycollectionstate(value::AbstractString) + return _nestedwritekeyindexedstate(codeunits(value)) +end + +function _nestedwritekeycollectionstate(value::JSONValue) + return _nestedwritekeyindexedstate(value.bytes) +end + +function _nestedwritekeycollectionstate(value::BSONValue) + return _nestedwritekeyindexedstate(value.bytes) +end + +function _nestedwritekeyvalidatedcount(value, shape, + state::Union{Nothing,_NestedWriteKeyCollectionState}, limits::Limits) + state === nothing && return + iscontainer = value isa Union{Pair,NamedTuple,StructValue,MapValue, + AbstractDict} + iscontainer |= value isa Tuple && + !_nestedwriteisfixedtuple(typeof(value)) + iscontainer |= value isa AbstractVector && + !(value isa AbstractVector{UInt8}) + iscontainer || return + _checklimit(:container_elements, state.count, + limits.max_container_elements) + return +end + +function _nestedwritekeydecimalpreflight(shape, value::Decimal, + limits::Limits) + digits = _decimaldigits(value.unscaled) + digits <= typemax(Int32) || throw(ArgumentError( + "DECIMAL MAP-key precision exceeds Int32")) + precision = max(Int32(1), Int32(digits), value.scale) + if shape isa _NestedWriteLeafShape && + shape.explicit isa _DecimalLogicalColumnSpec + spec = shape.explicit + value.scale == spec.scale || throw(ArgumentError( + "DECIMAL MAP key requires scale $(spec.scale), got $(value.scale)")) + _checkdecimalvalue(value.unscaled, spec.precision, shape.name, + ArgumentError) + precision = spec.precision + end + precision > 18 || return + width = _decimalwritewidth(precision, limits) + _checklimit(:decimal_bytes, width, limits.max_decimal_bytes) + _twoscomplementwidth(value.unscaled) <= width || throw(ArgumentError( + "DECIMAL MAP key does not fit its declared byte width")) + return +end + +function _nestedwritekeyleafpreflight(shape, value, + state::Union{Nothing,_NestedWriteKeyCollectionState}, limits::Limits) + ismissing(value) && return + if shape isa _NestedWriteLeafShape && shape.fixed_width !== nothing + width = Int(shape.fixed_width) + _checklimit(:string_bytes, width, limits.max_string_bytes) + state === nothing || state.count == width || throw(ArgumentError( + "fixed byte-array MAP key has a value with the wrong width")) + return + elseif value isa AbstractString + state === nothing && throw(AssertionError( + "string MAP key has no captured collection state")) + _checklimit(:string_bytes, state.count, limits.max_string_bytes) + elseif value isa AbstractVector{UInt8} + state === nothing && throw(AssertionError( + "byte MAP key has no captured collection state")) + _checklimit(:string_bytes, state.count, limits.max_string_bytes) + elseif value isa JSONValue + state === nothing && throw(AssertionError( + "JSON MAP key has no captured collection state")) + _checklimit(:string_bytes, state.count, limits.max_string_bytes) + elseif value isa BSONValue + state === nothing && throw(AssertionError( + "BSON MAP key has no captured collection state")) + _checklimit(:string_bytes, state.count, limits.max_string_bytes) + elseif value isa Decimal + _nestedwritekeydecimalpreflight(shape, value, limits) + elseif value isa Tuple && _nestedwriteisfixedtuple(typeof(value)) + state === nothing && throw(AssertionError( + "fixed-byte MAP key has no captured collection state")) + _checklimit(:string_bytes, state.count, limits.max_string_bytes) + end + return +end + +function _nestedwritekeyvectorcheck(value, + state::_NestedWriteKeyCollectionState) + state.indexed || throw(AssertionError( + "nested Parquet MAP key has no captured axes")) + value === state.source || throw(AssertionError( + "nested Parquet MAP key checked a different captured source")) + count = value isa AbstractVector ? + _nestedvectorcount(value, "nested Parquet MAP key") : length(value) + first, last = value isa AbstractVector ? + _nestedvectoraxes(value, "nested Parquet MAP key") : + (firstindex(value), lastindex(value)) + count == state.count && first == state.first && last == state.last || throw(ArgumentError( + "nested Parquet MAP key changed length or axes while it was copied")) + return +end + +function _nestedwritekeyshape(shape::Nothing, ::Int) + return nothing +end + +function _nestedwritekeyshape(shape::_NestedWriteStructShape, index::Int) + index <= length(shape.children) || throw(ArgumentError( + "nested Parquet MAP key changed its struct field count")) + return shape.children[index] +end + +function _nestedwritekeybytescalar(byte) + return try + UInt8(byte) + catch error + error isa Union{InexactError,MethodError} || rethrow() + throw(ArgumentError( + "nested Parquet MAP key contains a value that is not a UInt8")) + end +end + +function _nestedwritekeychildren(trace::_NestedWriteTrace, count::Int) + charge = _materializedarraybytes(_NestedWriteKeySnapshot, count) + _nestedwritetracereserve!(trace, charge) + return Vector{_NestedWriteKeySnapshot}(undef, count) +end + +function _nestedwritekeypaircount(count::Int, trace::_NestedWriteTrace) + count = try + _materializedproduct(count, 2) + catch error + error isa LimitError || rethrow() + throw(LimitError(:materialized_bytes, typemax(Int64), + trace.budget.maximum)) + end + count <= typemax(Int) || throw(LimitError(:materialized_bytes, + typemax(Int64), trace.budget.maximum)) + return Int(count) +end + +function _nestedwritekeynode!(trace::_NestedWriteTrace) + _nestedwritetracereserve!(trace, _MATERIALIZED_OBJECT_BYTES) + return +end + +function _nestedwritekeycopybytes(value, + state::_NestedWriteKeyCollectionState, action::String) + _nestedwritekeyvectorcheck(value, state) + output = Vector{UInt8}(undef, state.count) + position = 0 + for byte in value + _nestedwritekeyvectorcheck(value, state) + position += 1 + position <= length(output) || throw(ArgumentError( + "nested Parquet MAP key changed length while it was $action")) + output[position] = _nestedwritekeybytescalar(byte) + end + _nestedwritekeyvectorcheck(value, state) + position == length(output) || throw(ArgumentError( + "nested Parquet MAP key changed length while it was $action")) + return output +end + +function _nestedwritekeybytes(value, trace::_NestedWriteTrace, + state::_NestedWriteKeyCollectionState) + charge = _materializedarraybytes(UInt8, state.count) + _nestedwritetracereserve!(trace, charge) + return _nestedwritekeycopybytes(value, state, "copied") +end + +function _nestedwritekeysnapshot(value, trace::_NestedWriteTrace, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits) + return _nestedwritekeysnapshotiterative(value, trace, shape, limits, + nothing, nothing) +end + +function _nestedwritekeysnapshot(value, trace::_NestedWriteTrace, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits, witness) + return _nestedwritekeysnapshotiterative(value, trace, shape, limits, + witness, nothing) +end + +function _nestedwritekeysnapshot(value, trace::_NestedWriteTrace, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits, witness, + row_witness) + return _nestedwritekeysnapshotiterative(value, trace, shape, limits, + witness, row_witness) +end + +function _nestedwritekeysnapshotnode(value::AbstractString, + trace::_NestedWriteTrace, ::Limits, + ::Union{Nothing,_NestedWriteShape}, ::Int, + ::Nothing, + state::_NestedWriteKeyCollectionState) + source = state.source + _nestedwritekeynode!(trace) + bytes = _nestedwritekeybytes(source, trace, state) + isvalid(String, bytes) || throw(ArgumentError( + "Parquet MAP key contains invalid UTF-8")) + return _NestedWriteKeySnapshot(_NESTED_WRITE_KEY_STRING, typeof(value), + state.first, state.last, bytes, nothing, _NESTED_WRITE_EMPTY_KEYS) +end + +function _nestedwritekeysnapshotnode(value::AbstractVector{UInt8}, + trace::_NestedWriteTrace, ::Limits, + ::Union{Nothing,_NestedWriteShape}, ::Int, + ::Nothing, + state::_NestedWriteKeyCollectionState) + _nestedwritekeynode!(trace) + bytes = _nestedwritekeybytes(value, trace, state) + return _NestedWriteKeySnapshot(_NESTED_WRITE_KEY_BYTES, typeof(value), + state.first, state.last, bytes, nothing, + _NESTED_WRITE_EMPTY_KEYS) +end + +function _nestedwritekeysnapshotnode(value::JSONValue, + trace::_NestedWriteTrace, ::Limits, + ::Union{Nothing,_NestedWriteShape}, ::Int, + ::Nothing, + state::_NestedWriteKeyCollectionState) + _nestedwritekeynode!(trace) + bytes = _nestedwritekeybytes(state.source, trace, state) + return _NestedWriteKeySnapshot(_NESTED_WRITE_KEY_BYTES, JSONValue, + 1, length(bytes), bytes, nothing, _NESTED_WRITE_EMPTY_KEYS) +end + +function _nestedwritekeysnapshotnode(value::BSONValue, + trace::_NestedWriteTrace, ::Limits, + ::Union{Nothing,_NestedWriteShape}, ::Int, + ::Nothing, + state::_NestedWriteKeyCollectionState) + _nestedwritekeynode!(trace) + bytes = _nestedwritekeybytes(state.source, trace, state) + return _NestedWriteKeySnapshot(_NESTED_WRITE_KEY_BYTES, BSONValue, + 1, length(bytes), bytes, nothing, _NESTED_WRITE_EMPTY_KEYS) +end + +function _nestedwritekeysnapshotnode(value::Decimal, + trace::_NestedWriteTrace, ::Limits, + ::Union{Nothing,_NestedWriteShape}, ::Int, + ::Nothing, ::Nothing) + _nestedwritekeynode!(trace) + bits = ndigits(value.unscaled; base=2) + payload = cld(bits, 8) + charge = _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _materializedarraybytes(UInt8, payload)) + _nestedwritetracereserve!(trace, charge) + return _NestedWriteKeySnapshot(_NESTED_WRITE_KEY_SCALAR, Decimal, 1, 1, + copy(value), nothing, _NESTED_WRITE_EMPTY_KEYS) +end + +function _nestedwritekeysnapshotnode(value::Tuple, trace::_NestedWriteTrace, + ::Limits, shape::Union{Nothing,_NestedWriteShape}, ::Int, ::Nothing, + state::_NestedWriteKeyCollectionState) + _nestedwriteisfixedtuple(typeof(value)) || throw(AssertionError( + "recursive Tuple MAP keys require the iterative frame engine")) + shape isa Union{Nothing,_NestedWriteLeafShape} || throw(ArgumentError( + "nested Parquet MAP key does not match its declared leaf topology")) + shape === nothing || _nestedwriteisfixedtuple(shape.value_type) || + throw(ArgumentError( + "nested Parquet MAP key does not match its declared leaf topology")) + _nestedwritekeynode!(trace) + bytes = _nestedwritekeybytes(value, trace, state) + return _NestedWriteKeySnapshot(_NESTED_WRITE_KEY_BYTES, typeof(value), + state.first, state.last, bytes, nothing, _NESTED_WRITE_EMPTY_KEYS) +end + +function _nestedwritekeysnapshotnode(value, trace::_NestedWriteTrace, + ::Limits, shape::Union{Nothing,_NestedWriteShape}, ::Int, + ::Nothing, ::Nothing) + shape isa Union{Nothing,_NestedWriteLeafShape} || throw(ArgumentError( + "nested Parquet MAP key does not match its declared leaf topology")) + value === nothing && throw(ArgumentError( + "nothing is not a stable Parquet MAP key")) + type = typeof(value) + Base.ismutabletype(type) && throw(ArgumentError( + "Parquet MAP key type $type does not have stable copy semantics")) + (isbitstype(type) || fieldcount(type) == 0) || throw(ArgumentError( + "Parquet MAP key type $type does not have stable copy semantics")) + _nestedwritekeynode!(trace) + return _NestedWriteKeySnapshot(_NESTED_WRITE_KEY_SCALAR, type, 1, 1, + value, nothing, _NESTED_WRITE_EMPTY_KEYS) +end + +function _nestedwritekeybytesequal(expected::Vector{UInt8}, current) + state = _nestedwritekeyindexedstate(current) + return _nestedwritekeybytesequal(expected, current, state) +end + +function _nestedwritekeybytesequal(expected::Vector{UInt8}, current, + state::_NestedWriteKeyCollectionState) + length(expected) == state.count || return false + _nestedwritekeyvectorcheck(current, state) + position = 0 + for byte in current + _nestedwritekeyvectorcheck(current, state) + position += 1 + position <= length(expected) || return false + expected[position] == _nestedwritekeybytescalar(byte) || return false + end + _nestedwritekeyvectorcheck(current, state) + return position == length(expected) +end + +function _nestedwritekeyequal(expected::_NestedWriteKeySnapshot, current, + trace::_NestedWriteTrace, limits::Limits) + return _nestedwritekeyequaliterative(expected, current, trace, limits, + nothing, nothing, nothing) +end + +function _nestedwritekeyequal(expected::_NestedWriteKeySnapshot, current, + trace::_NestedWriteTrace, limits::Limits, shape, witness, row_witness) + return _nestedwritekeyequaliterative(expected, current, trace, limits, + shape, witness, row_witness) +end + +function _nestedwritekeycapturecheck(frame::_NestedWriteKeyCaptureFrame) + _nestedwriterowcheck(frame.row_witness) + value = frame.value + kind = frame.kind + if kind == _NESTED_WRITE_KEY_STRUCT + value.children isa Vector{AbstractVector} && + length(value.names) == length(frame.children) && + length(value.children) == length(frame.children) || + throw(ArgumentError( + "nested Parquet MAP key changed its struct backing metadata while it was copied")) + elseif kind == _NESTED_WRITE_KEY_LIST + value isa ListValue && _validatelistvalue(value) + _nestedwritekeyvectorcheck(value, frame.state) + elseif kind == _NESTED_WRITE_KEY_MAP + if value isa MapValue + _validatemapvalue(value) + _nestedwritekeyvectorcheck(value, frame.state) + end + end + return +end + +function _nestedwritekeycaptureframe(value, trace::_NestedWriteTrace, + limits::Limits, shape::Union{Nothing,_NestedWriteShape}, depth::Int, + parent::Union{Nothing,_NestedWriteKeyCaptureFrame}, parent_slot::Int, + state::_NestedWriteKeyCollectionState, witness, row_witness) + count = state.count + kind = UInt8(0) + names = nothing + childcount = count + if value isa Pair + shape === nothing || throw(ArgumentError( + "nested Parquet MAP key does not match its declared topology")) + count == 2 || throw(ArgumentError( + "nested Parquet MAP key changed its Pair length")) + kind = _NESTED_WRITE_KEY_PAIR + elseif value isa NamedTuple + shape isa Union{Nothing,_NestedWriteStructShape} || throw(ArgumentError( + "nested Parquet MAP key does not match its declared struct topology")) + shape === nothing || length(shape.children) == count || throw( + ArgumentError("nested Parquet MAP key changed its struct field count")) + kind = _NESTED_WRITE_KEY_NAMED_TUPLE + names = keys(value) + elseif value isa Tuple + shape === nothing || throw(ArgumentError( + "nested Parquet MAP key does not match its declared topology")) + kind = _NESTED_WRITE_KEY_TUPLE + elseif value isa StructValue + shape isa Union{Nothing,_NestedWriteStructShape} || throw(ArgumentError( + "nested Parquet MAP key does not match its declared struct topology")) + _validatestructvalue(value) + if shape !== nothing + value.names == shape.names || throw(ArgumentError( + "nested Parquet MAP key changed its struct field names")) + count == length(shape.children) || throw(ArgumentError( + "nested Parquet MAP key changed its struct field count")) + end + namesbefore = value.names + length(namesbefore) == count || throw(ArgumentError( + "nested Parquet MAP key changed its struct field names")) + names = namesbefore + kind = _NESTED_WRITE_KEY_STRUCT + elseif value isa MapValue + shape isa Union{Nothing,_NestedWriteMapShape} || throw(ArgumentError( + "nested Parquet MAP key does not match its declared MAP topology")) + _validatemapvalue(value) + kind = _NESTED_WRITE_KEY_MAP + childcount = _nestedwritekeypaircount(count, trace) + elseif value isa AbstractDict + shape isa Union{Nothing,_NestedWriteMapShape} || throw(ArgumentError( + "nested Parquet MAP key does not match its declared MAP topology")) + kind = _NESTED_WRITE_KEY_MAP + childcount = _nestedwritekeypaircount(count, trace) + elseif value isa AbstractVector + shape isa Union{Nothing,_NestedWriteListShape} || throw(ArgumentError( + "nested Parquet MAP key does not match its declared LIST topology")) + value isa ListValue && _validatelistvalue(value) + kind = _NESTED_WRITE_KEY_LIST + else + throw(AssertionError("unknown recursive nested writer MAP-key type")) + end + _nestedwritekeychilddepth(count, depth, limits) + _nestedwritekeycaptureactive(value, parent) && throw(ArgumentError( + "cyclic nested Parquet MAP key")) + container_snapshot = kind == _NESTED_WRITE_KEY_LIST && + _nestedwriteispackagevector(value) ? + _nestedwriteoccurrencesnapshot!(trace, value, shape, limits) : nothing + _nestedwritekeyframecharge!(trace) + try + frame = _NestedWriteKeyCaptureFrame(parent, parent_slot, value, shape, + witness, row_witness, state, depth, kind, + _NESTED_WRITE_EMPTY_KEYS, nothing, 0, nothing, nothing, nothing, + nothing, nothing, nothing, nothing) + frame.pending = container_snapshot + _nestedwritekeynode!(trace) + frame.children = _nestedwritekeychildren(trace, childcount) + if kind == _NESTED_WRITE_KEY_STRUCT + namecharge = _materializedarraybytes(String, count) + _nestedwritetracereserve!(trace, namecharge) + frame.names = copy(names) + else + frame.names = names + end + frame.iterator = if kind == _NESTED_WRITE_KEY_LIST + eachindex(value) + elseif kind == _NESTED_WRITE_KEY_MAP && value isa AbstractDict + state.source.first + else + nothing + end + if iszero(childcount) + snapshot = _nestedwritekeycapturefinish(frame, trace) + _nestedwritekeycaptureframefree!(trace, frame) + return snapshot + end + return frame + catch + _nestedwritekeycollectionfree!(trace, state) + _nestedwritekeyframefree!(trace) + rethrow() + end +end + +function _nestedwritekeycapturestart(value, trace::_NestedWriteTrace, + limits::Limits, shape::Union{Nothing,_NestedWriteShape}, depth::Int, + parent::Union{Nothing,_NestedWriteKeyCaptureFrame}, parent_slot::Int, + witness, row_witness) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + _nestedwriterowcheck(row_witness) + _nestedwritekeywitnessroot(witness, value) + _nestedwritekeydeclared(shape, value) + if value isa AbstractDict + _nestedwritekeycaptureactive(value, parent) && throw(ArgumentError( + "cyclic nested Parquet MAP key")) + keyshape = shape isa _NestedWriteMapShape ? shape.key : nothing + valueshape = shape isa _NestedWriteMapShape ? shape.value : nothing + materialization = _nestedwritedictmaterialize(trace, value, keyshape, + valueshape, limits, depth) + state = _NestedWriteKeyCollectionState(materialization.count, 0, 0, + false, materialization) + try + _nestedwritekeyvalidatedcount(value, shape, state, limits) + _nestedwritekeyleafpreflight(shape, value, state, limits) + catch + _nestedwritedictrelease!(trace, materialization) + rethrow() + end + try + return _nestedwritekeycaptureframe(value, trace, limits, shape, + depth, parent, parent_slot, state, witness, row_witness) + catch + _nestedwritedictrelease!(trace, materialization) + rethrow() + end + end + state = _nestedwritekeycollectionstate(value) + _nestedwritekeyvalidatedcount(value, shape, state, limits) + _nestedwritekeyleafpreflight(shape, value, state, limits) + if !_nestedwritekeyrecursive(value) + return _nestedwritekeysnapshotnode(value, trace, limits, shape, + depth, nothing, state) + end + state === nothing && throw(AssertionError( + "recursive nested writer MAP key has no collection state")) + return _nestedwritekeycaptureframe(value, trace, limits, shape, depth, + parent, parent_slot, state, witness, row_witness) +end + +function _nestedwritekeycapturesetnext!(frame::_NestedWriteKeyCaptureFrame, + value, shape::Union{Nothing,_NestedWriteShape}, witness, row_witness) + frame.position += 1 + frame.position <= length(frame.children) || throw(ArgumentError( + "nested Parquet MAP key changed length while it was copied")) + frame.next_value = value + frame.next_shape = shape + frame.next_witness = witness + frame.next_row_witness = row_witness + return true +end + +function _nestedwritekeycaptureadvance!(frame::_NestedWriteKeyCaptureFrame, + trace::_NestedWriteTrace, limits::Limits) + _nestedwritekeycapturecheck(frame) + frame.position == length(frame.children) && return false + value = frame.value + shape = frame.shape + kind = frame.kind + if kind == _NESTED_WRITE_KEY_PAIR + child = iszero(frame.position) ? value.first : value.second + return _nestedwritekeycapturesetnext!(frame, child, nothing, nothing, + nothing) + elseif kind == _NESTED_WRITE_KEY_TUPLE + index = frame.position + 1 + child = value[index] + return _nestedwritekeycapturesetnext!(frame, child, nothing, nothing, + nothing) + elseif kind == _NESTED_WRITE_KEY_NAMED_TUPLE + index = frame.position + 1 + childshape = _nestedwritekeyshape(shape, index) + child = getfield(value, index) + return _nestedwritekeycapturesetnext!(frame, child, childshape, + nothing, nothing) + elseif kind == _NESTED_WRITE_KEY_STRUCT + index = frame.position + 1 + childshape = _nestedwritekeyshape(shape, index) + expectedchild = if shape isa _NestedWriteStructShape + something(shape.source_children)[index] + else + _nestedwritekeywitnesssource(frame.witness, index) + end + childvector = _nestedwritestructaccesschild(value, value.names, + value.children, length(frame.children), index, + frame.names[index], expectedchild) + childwitness = _nestedwritekeywitnesschild(frame.witness, index) + child, row_witness = _nestedwriterowaccess(childvector, value.index, + _nestedwritekeyrowsnapshot(childshape, childwitness, trace, + childvector, limits)) + return _nestedwritekeycapturesetnext!(frame, child, childshape, + childwitness, row_witness) + elseif kind == _NESTED_WRITE_KEY_LIST + result = iszero(frame.position) ? iterate(frame.iterator) : + iterate(frame.iterator, frame.iteration) + result === nothing && throw(ArgumentError( + "nested Parquet MAP key changed length while it was copied")) + index, iteration = result + frame.iteration = iteration + childshape = shape === nothing ? nothing : shape.element + childwitness = _nestedwritekeywitnesslist(frame.witness) + if value isa ListValue + physical = value.first + index - 1 + child, row_witness = _nestedwriterowaccess(value.values, physical, + _nestedwritekeyrowsnapshot(childshape, childwitness, trace, + value.values, limits)) + elseif frame.pending isa _NestedWriteVectorSnapshot + child, row_witness = _nestedwriterowaccess(value, index, + frame.pending) + else + child = _nestedwritelistaccessitem(value, index) + row_witness = nothing + end + _nestedwritekeycapturecheck(frame) + return _nestedwritekeycapturesetnext!(frame, child, childshape, + childwitness, row_witness) + elseif kind == _NESTED_WRITE_KEY_MAP && value isa MapValue + if iseven(frame.position) + entry = frame.position ÷ 2 + 1 + childshape = shape === nothing ? nothing : shape.key + childwitness = _nestedwritekeywitnessmapkey(frame.witness) + physical = value.first + entry - 1 + child, row_witness = _nestedwriterowaccess(value.keys, physical, + _nestedwritekeyrowsnapshot(childshape, childwitness, trace, + value.keys, limits)) + frame.pending = physical + return _nestedwritekeycapturesetnext!(frame, child, + childshape, childwitness, row_witness) + end + childshape = shape === nothing ? nothing : shape.value + childwitness = _nestedwritekeywitnessmapvalue(frame.witness) + if value.values === nothing + child = missing + row_witness = nothing + else + child, row_witness = _nestedwriterowaccess(value.values, + frame.pending, _nestedwritekeyrowsnapshot(childshape, + childwitness, trace, value.values, limits)) + end + return _nestedwritekeycapturesetnext!(frame, child, childshape, + childwitness, row_witness) + elseif kind == _NESTED_WRITE_KEY_MAP + if iseven(frame.position) + entry = frame.iterator + entry isa _NestedWriteDictEntry || throw(ArgumentError( + "nested Parquet MAP key changed length while it was copied")) + frame.iterator = entry.next + frame.pending = entry + childshape = shape === nothing ? nothing : shape.key + childwitness = _nestedwritekeywitnessmapkey(frame.witness) + return _nestedwritekeycapturesetnext!(frame, entry.key, + childshape, childwitness, nothing) + end + entry = frame.pending + entry isa _NestedWriteDictEntry || throw(ArgumentError( + "nested Parquet MAP key changed its entry while it was copied")) + childshape = shape === nothing ? nothing : shape.value + childwitness = _nestedwritekeywitnessmapvalue(frame.witness) + return _nestedwritekeycapturesetnext!(frame, entry.value, childshape, + childwitness, nothing) + end + throw(AssertionError("unknown recursive nested writer MAP-key kind")) +end + +function _nestedwritekeycapturefinish(frame::_NestedWriteKeyCaptureFrame, + trace::_NestedWriteTrace) + frame.position == length(frame.children) || throw(ArgumentError( + "nested Parquet MAP key changed length while it was copied")) + if frame.kind == _NESTED_WRITE_KEY_MAP && frame.value isa AbstractDict + frame.iterator === nothing || throw( + ArgumentError("nested Parquet MAP key changed length while it was copied")) + elseif frame.kind == _NESTED_WRITE_KEY_LIST + result = iszero(frame.position) ? iterate(frame.iterator) : + iterate(frame.iterator, frame.iteration) + result === nothing || throw( + ArgumentError("nested Parquet MAP key changed length while it was copied")) + end + stored = if frame.kind == _NESTED_WRITE_KEY_MAP + true + elseif frame.kind == _NESTED_WRITE_KEY_STRUCT && frame.witness !== nothing + frame.witness + elseif frame.kind == _NESTED_WRITE_KEY_STRUCT && + frame.shape isa _NestedWriteStructShape + frame.shape.source_children + else + nothing + end + return _NestedWriteKeySnapshot(frame.kind, typeof(frame.value), + frame.kind in (_NESTED_WRITE_KEY_LIST, _NESTED_WRITE_KEY_MAP) ? + frame.state.first : 1, + frame.kind in (_NESTED_WRITE_KEY_LIST, _NESTED_WRITE_KEY_MAP) ? + frame.state.last : length(frame.children), + stored, frame.names, frame.children) +end + +function _nestedwritekeycapturecleanup!(trace::_NestedWriteTrace, + frame::Union{Nothing,_NestedWriteKeyCaptureFrame}) + current = frame + while current !== nothing + parent = current.parent + _nestedwritekeycaptureframefree!(trace, current) + current = parent + end + return +end + +function _nestedwritekeysnapshotiterative(value, trace::_NestedWriteTrace, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits, witness, + row_witness) + started = _nestedwritekeycapturestart(value, trace, limits, shape, 1, + nothing, 0, witness, row_witness) + started isa _NestedWriteKeySnapshot && return started + current = started::_NestedWriteKeyCaptureFrame + try + while true + if _nestedwritekeycaptureadvance!(current, trace, limits) + depth = _nestedwritekeynextdepth(current.depth, limits) + started = _nestedwritekeycapturestart(current.next_value, + trace, limits, current.next_shape, depth, current, + current.position, current.next_witness, + current.next_row_witness) + if started isa _NestedWriteKeySnapshot + current.children[current.position] = started + _nestedwriterowcheck(current.next_row_witness) + else + current = started::_NestedWriteKeyCaptureFrame + end + continue + end + snapshot = _nestedwritekeycapturefinish(current, trace) + parent = current.parent + slot = current.parent_slot + _nestedwritekeycaptureframefree!(trace, current) + if parent === nothing + current = nothing + return snapshot + end + _nestedwriterowcheck(current.row_witness) + current = parent + current.children[slot] = snapshot + end + finally + _nestedwritekeycapturecleanup!(trace, current) + end +end + +function _nestedwritekeycomparebytes(expected::_NestedWriteKeySnapshot, + current) + source = if current isa AbstractString + codeunits(current) + elseif current isa Union{JSONValue,BSONValue} + current.bytes + else + current + end + state = _nestedwritekeyindexedstate(source) + length(expected.value) == state.count && + expected.first == state.first && expected.last == state.last || throw( + ArgumentError( + "nested Parquet MAP key changed length or axes while it was compared")) + return _nestedwritekeybytesequal(expected.value, source, state) +end + +function _nestedwritekeycompareleaf(expected::_NestedWriteKeySnapshot, + current, limits::Limits) + kind = expected.kind + if kind == _NESTED_WRITE_KEY_SCALAR + typeof(current) === expected.source_type || return false + return isequal(expected.value, current) + elseif kind == _NESTED_WRITE_KEY_STRING + current isa AbstractString || return false + typeof(current) === expected.source_type || return false + return _nestedwritekeycomparebytes(expected, current) + elseif kind == _NESTED_WRITE_KEY_BYTES + if expected.source_type === JSONValue + current isa JSONValue || return false + elseif expected.source_type === BSONValue + current isa BSONValue || return false + else + current isa Union{AbstractVector{UInt8},Tuple} || return false + typeof(current) === expected.source_type || return false + end + return _nestedwritekeycomparebytes(expected, current) + end + return nothing +end + +function _nestedwritekeycomparecheck(frame::_NestedWriteKeyCompareFrame) + _nestedwriterowcheck(frame.row_witness) + expected = frame.expected + current = frame.current + kind = expected.kind + if kind == _NESTED_WRITE_KEY_STRUCT + current.names == expected.names && + length(current.children) == length(expected.children) || return false + elseif kind == _NESTED_WRITE_KEY_LIST + current isa ListValue && _validatelistvalue(current) + _nestedwritekeyvectorcheck(current, frame.state) + frame.state.first == expected.first && + frame.state.last == expected.last || return false + elseif kind == _NESTED_WRITE_KEY_MAP + if current isa MapValue + _validatemapvalue(current) + _nestedwritekeyvectorcheck(current, frame.state) + frame.state.first == expected.first && + frame.state.last == expected.last || return false + end + end + return true +end + +function _nestedwritekeycompareinitialize!(frame::_NestedWriteKeyCompareFrame, + limits::Limits) + expected = frame.expected + current = frame.current + kind = expected.kind + childcount = length(expected.children) + if kind == _NESTED_WRITE_KEY_PAIR + current isa Pair || return false + typeof(current) === expected.source_type || return false + elseif kind == _NESTED_WRITE_KEY_TUPLE + current isa Tuple || return false + typeof(current) === expected.source_type || return false + elseif kind == _NESTED_WRITE_KEY_NAMED_TUPLE + current isa NamedTuple || return false + typeof(current) === expected.source_type || return false + keys(current) == expected.names || return false + elseif kind == _NESTED_WRITE_KEY_STRUCT + current isa StructValue || return false + expected.source_type === StructValue || return false + _validatestructvalue(current) + current.names == expected.names || return false + elseif kind == _NESTED_WRITE_KEY_LIST + current isa AbstractVector || return false + typeof(current) === expected.source_type || return false + current isa ListValue && _validatelistvalue(current) + elseif kind == _NESTED_WRITE_KEY_MAP + current isa Union{MapValue,AbstractDict} || return false + typeof(current) === expected.source_type || return false + iseven(childcount) || return false + else + return false + end + state = current isa AbstractDict ? frame.state : + _nestedwritekeycollectionstate(current) + state === nothing && return false + _nestedwritekeyvalidatedcount(current, nothing, state, limits) + if kind == _NESTED_WRITE_KEY_PAIR + state.count == 2 == childcount || return false + elseif kind in (_NESTED_WRITE_KEY_TUPLE, + _NESTED_WRITE_KEY_NAMED_TUPLE, _NESTED_WRITE_KEY_STRUCT, + _NESTED_WRITE_KEY_LIST) + state.count == childcount || return false + else + state.count == childcount ÷ 2 || return false + end + if kind == _NESTED_WRITE_KEY_LIST + state.first == expected.first && state.last == expected.last || + return false + elseif kind == _NESTED_WRITE_KEY_MAP && current isa MapValue + _validatemapvalue(current) + state.first == expected.first && state.last == expected.last || + return false + end + frame.state = state + frame.iterator = if kind == _NESTED_WRITE_KEY_LIST + eachindex(current) + elseif kind == _NESTED_WRITE_KEY_MAP && current isa AbstractDict + state.source.first + else + nothing + end + return true +end + +function _nestedwritekeycompareframe(expected::_NestedWriteKeySnapshot, + current, trace::_NestedWriteTrace, limits::Limits, depth::Int, + parent::Union{Nothing,_NestedWriteKeyCompareFrame}, shape, witness, + row_witness) + recursive = _nestedwritekeyrecursive(current) + recursive || return false + isempty(expected.children) || + _nestedwritekeychilddepth(length(expected.children), depth, limits) + _nestedwritekeycompareactive(current, parent) && throw(ArgumentError( + "cyclic nested Parquet MAP key")) + container_snapshot = expected.kind == _NESTED_WRITE_KEY_LIST && + _nestedwriteispackagevector(current) ? + _nestedwriteoccurrencesnapshot!(trace, current, shape, limits) : nothing + materialization = if current isa AbstractDict + keyshape = shape isa _NestedWriteMapShape ? shape.key : nothing + valueshape = shape isa _NestedWriteMapShape ? shape.value : nothing + _nestedwritedictmaterialize(trace, current, keyshape, valueshape, + limits, depth) + else + nothing + end + state = materialization === nothing ? + _NestedWriteKeyCollectionState(0, 0, 0, false, nothing) : + _NestedWriteKeyCollectionState(materialization.count, 0, 0, false, + materialization) + try + _nestedwritekeyframecharge!(trace) + catch + _nestedwritekeycollectionfree!(trace, state) + rethrow() + end + try + frame = _NestedWriteKeyCompareFrame(parent, expected, current, shape, + witness, row_witness, state, depth, 0, nothing, nothing, nothing, + nothing, nothing, nothing, nothing, nothing) + frame.pending = container_snapshot + if !_nestedwritekeycompareinitialize!(frame, limits) + _nestedwritekeycompareframefree!(trace, frame) + return false + end + return frame + catch + _nestedwritekeycollectionfree!(trace, state) + _nestedwritekeyframefree!(trace) + rethrow() + end +end + +function _nestedwritekeyfamilymatches(expected::_NestedWriteKeySnapshot, + current) + kind = expected.kind + if kind == _NESTED_WRITE_KEY_SCALAR + return !_nestedwritekeyrecursive(current) && + typeof(current) === expected.source_type + elseif kind == _NESTED_WRITE_KEY_STRING + return current isa AbstractString && + typeof(current) === expected.source_type + elseif kind == _NESTED_WRITE_KEY_BYTES + return !_nestedwritekeyrecursive(current) && + typeof(current) === expected.source_type + elseif kind == _NESTED_WRITE_KEY_PAIR + return current isa Pair && typeof(current) === expected.source_type + elseif kind == _NESTED_WRITE_KEY_TUPLE + return current isa Tuple && typeof(current) === expected.source_type + elseif kind == _NESTED_WRITE_KEY_NAMED_TUPLE + return current isa NamedTuple && + typeof(current) === expected.source_type + elseif kind == _NESTED_WRITE_KEY_STRUCT + return current isa StructValue + elseif kind == _NESTED_WRITE_KEY_LIST + return current isa AbstractVector && + !(current isa AbstractVector{UInt8}) && + typeof(current) === expected.source_type + elseif kind == _NESTED_WRITE_KEY_MAP + return current isa Union{MapValue,AbstractDict} && + typeof(current) === expected.source_type + end + return false +end + +function _nestedwritekeycomparestart(expected::_NestedWriteKeySnapshot, + current, trace::_NestedWriteTrace, limits::Limits, depth::Int, + parent::Union{Nothing,_NestedWriteKeyCompareFrame}, shape, witness, + row_witness) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + _nestedwriterowcheck(row_witness) + _nestedwritekeywitnessroot(witness, current) + shape === nothing || _nestedwritekeydeclared(shape, current) + _nestedwritekeyfamilymatches(expected, current) || return false + _nestedwritekeyrecursive(current) && return _nestedwritekeycompareframe( + expected, current, trace, limits, depth, parent, shape, witness, + row_witness) + leaf = _nestedwritekeycompareleaf(expected, current, limits) + return leaf === nothing ? false : leaf +end + +function _nestedwritekeycomparesetnext!(frame::_NestedWriteKeyCompareFrame, + current, shape, witness, row_witness) + frame.position += 1 + frame.position <= length(frame.expected.children) || return false + frame.next_expected = frame.expected.children[frame.position] + frame.next_current = current + frame.next_shape = shape + frame.next_witness = witness + frame.next_row_witness = row_witness + return true +end + +function _nestedwritekeycompareadvance!(frame::_NestedWriteKeyCompareFrame, + trace::_NestedWriteTrace, limits::Limits) + _nestedwritekeycomparecheck(frame) || return false + frame.position == length(frame.expected.children) && return nothing + current = frame.current + kind = frame.expected.kind + if kind == _NESTED_WRITE_KEY_PAIR + child = iszero(frame.position) ? current.first : current.second + return _nestedwritekeycomparesetnext!(frame, child, nothing, nothing, + nothing) + elseif kind == _NESTED_WRITE_KEY_TUPLE + return _nestedwritekeycomparesetnext!(frame, + current[frame.position + 1], nothing, nothing, nothing) + elseif kind == _NESTED_WRITE_KEY_NAMED_TUPLE + shape = frame.shape + childshape = shape === nothing ? nothing : + _nestedwritekeyshape(shape, frame.position + 1) + return _nestedwritekeycomparesetnext!(frame, + getfield(current, frame.position + 1), childshape, nothing, + nothing) + elseif kind == _NESTED_WRITE_KEY_STRUCT + index = frame.position + 1 + expectedchild = _nestedwritekeystoredsource(frame.expected.value, + index) + childvector = _nestedwritestructaccesschild(current, current.names, + current.children, length(frame.expected.children), index, + frame.expected.names[index], expectedchild) + childshape = frame.shape === nothing ? nothing : + _nestedwritekeyshape(frame.shape, index) + childwitness = _nestedwritekeywitnesschild(frame.witness, index) + child, row_witness = _nestedwriterowaccess(childvector, current.index, + _nestedwritekeyrowsnapshot(childshape, childwitness, trace, + childvector, limits)) + return _nestedwritekeycomparesetnext!(frame, child, childshape, + childwitness, row_witness) + elseif kind == _NESTED_WRITE_KEY_LIST + result = iszero(frame.position) ? iterate(frame.iterator) : + iterate(frame.iterator, frame.iteration) + result === nothing && return false + index, iteration = result + frame.iteration = iteration + childshape = frame.shape === nothing ? nothing : frame.shape.element + childwitness = _nestedwritekeywitnesslist(frame.witness) + if current isa ListValue + physical = current.first + index - 1 + child, row_witness = _nestedwriterowaccess(current.values, + physical, _nestedwritekeyrowsnapshot(childshape, + childwitness, trace, current.values, limits)) + elseif frame.pending isa _NestedWriteVectorSnapshot + child, row_witness = _nestedwriterowaccess(current, index, + frame.pending) + else + child = _nestedwritelistaccessitem(current, index) + row_witness = nothing + end + _nestedwritekeycomparecheck(frame) || return false + return _nestedwritekeycomparesetnext!(frame, child, childshape, + childwitness, row_witness) + elseif kind == _NESTED_WRITE_KEY_MAP && current isa MapValue + if iseven(frame.position) + entry = frame.position ÷ 2 + 1 + childshape = frame.shape === nothing ? nothing : frame.shape.key + childwitness = _nestedwritekeywitnessmapkey(frame.witness) + physical = current.first + entry - 1 + child, row_witness = _nestedwriterowaccess(current.keys, + physical, _nestedwritekeyrowsnapshot(childshape, + childwitness, trace, current.keys, limits)) + frame.pending = physical + return _nestedwritekeycomparesetnext!(frame, child, childshape, + childwitness, row_witness) + end + childshape = frame.shape === nothing ? nothing : frame.shape.value + childwitness = _nestedwritekeywitnessmapvalue(frame.witness) + if current.values === nothing + child = missing + row_witness = nothing + else + child, row_witness = _nestedwriterowaccess(current.values, + frame.pending, _nestedwritekeyrowsnapshot(childshape, + childwitness, trace, current.values, limits)) + end + return _nestedwritekeycomparesetnext!(frame, child, childshape, + childwitness, row_witness) + elseif kind == _NESTED_WRITE_KEY_MAP + if iseven(frame.position) + entry = frame.iterator + entry isa _NestedWriteDictEntry || return false + frame.iterator = entry.next + frame.pending = entry + childshape = frame.shape === nothing ? nothing : frame.shape.key + childwitness = _nestedwritekeywitnessmapkey(frame.witness) + return _nestedwritekeycomparesetnext!(frame, entry.key, + childshape, childwitness, nothing) + end + entry = frame.pending + entry isa _NestedWriteDictEntry || return false + childshape = frame.shape === nothing ? nothing : frame.shape.value + childwitness = _nestedwritekeywitnessmapvalue(frame.witness) + return _nestedwritekeycomparesetnext!(frame, entry.value, + childshape, childwitness, nothing) + end + throw(AssertionError("unknown recursive nested writer MAP-key kind")) +end + +function _nestedwritekeycomparefinish(frame::_NestedWriteKeyCompareFrame, + trace::_NestedWriteTrace) + frame.position == length(frame.expected.children) || return false + if frame.expected.kind == _NESTED_WRITE_KEY_MAP && + frame.current isa AbstractDict + frame.iterator === nothing || return false + elseif frame.expected.kind == _NESTED_WRITE_KEY_LIST + result = iszero(frame.position) ? iterate(frame.iterator) : + iterate(frame.iterator, frame.iteration) + result === nothing || return false + end + return true +end + +function _nestedwritekeycomparecleanup!(trace::_NestedWriteTrace, + frame::Union{Nothing,_NestedWriteKeyCompareFrame}) + current = frame + while current !== nothing + parent = current.parent + _nestedwritekeycompareframefree!(trace, current) + current = parent + end + return +end + +function _nestedwritekeyequaliterative(expected::_NestedWriteKeySnapshot, + current, trace::_NestedWriteTrace, limits::Limits, shape, witness, + row_witness) + started = _nestedwritekeycomparestart(expected, current, trace, limits, 1, + nothing, shape, witness, row_witness) + started isa Bool && return started + frame = started::_NestedWriteKeyCompareFrame + try + while true + advanced = _nestedwritekeycompareadvance!(frame, trace, limits) + advanced === false && return false + if advanced === true + child = something(frame.next_expected) + depth = _nestedwritekeynextdepth(frame.depth, limits) + started = _nestedwritekeycomparestart(child, + frame.next_current, trace, limits, depth, frame, + frame.next_shape, frame.next_witness, + frame.next_row_witness) + started === false && return false + if started === true + _nestedwriterowcheck(frame.next_row_witness) + continue + end + frame = started::_NestedWriteKeyCompareFrame + continue + end + _nestedwritekeycomparefinish(frame, trace) || return false + parent = frame.parent + _nestedwritekeycompareframefree!(trace, frame) + if parent === nothing + frame = nothing + return true + end + _nestedwriterowcheck(frame.row_witness) + frame = parent + end + finally + _nestedwritekeycomparecleanup!(trace, frame) + end +end + +function _nestedwritetracekey!(::Nothing, value, + ::Union{Nothing,_NestedWriteShape}, ::Limits) + return nothing +end + +function _nestedwritetracekey!(::Nothing, value, + ::Union{Nothing,_NestedWriteShape}, ::Limits, witness) + return nothing +end + +function _nestedwritetracekey!(::Nothing, value, + ::Union{Nothing,_NestedWriteShape}, ::Limits, witness, row_witness) + return nothing +end + +function _nestedwritetracekey!(trace::_NestedWriteTrace, value, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits) + return _nestedwritetracekey!(trace, value, shape, limits, nothing) +end + +function _nestedwritetracekey!(trace::_NestedWriteTrace, value, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits, witness) + return _nestedwritetracekey!(trace, value, shape, limits, witness, + nothing) +end + +function _nestedwritetracekey!(trace::_NestedWriteTrace, value, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits, witness, + row_witness) + _nestedwriterowcheck(row_witness) + _nestedwritekeywitnessroot(witness, value) + _nestedwritekeydeclared(shape, value) + if trace.capturing + snapshot = _nestedwritekeysnapshot(value, trace, shape, limits, + witness, row_witness) + _nestedwritetracecapture!(trace, _NestedWriteTraceEvent( + _NESTED_WRITE_TRACE_KEY, Int64(0), Int64(0), Int64(0), Int64(0), + snapshot)) + return snapshot + end + expected = _nestedwritetracepeekkey(trace) + expected.kind == _NESTED_WRITE_TRACE_KEY && iszero(expected.a) && + iszero(expected.b) && iszero(expected.c) && iszero(expected.d) || + throw(ArgumentError( + "nested Parquet input changed its occurrence topology between writer passes")) + snapshot = expected.value::_NestedWriteKeySnapshot + _nestedwritekeyequal(snapshot, value, trace, limits, shape, witness, + row_witness) || + throw(ArgumentError( + "nested Parquet MAP keys changed content or order between writer passes")) + consumed = _nestedwritetracenext!(trace) + consumed === expected || throw(ArgumentError( + "nested Parquet input changed its MAP-key trace topology")) + return snapshot +end + +function _nestedwritetracekey!(trace, value) + return _nestedwritetracekey!(trace, value, nothing, Limits()) +end + +""" + _nestedwritekeyassert!(nothing, value, shape, limits, trace, witness, row_witness) + +Assert a MAP key when no key snapshot was captured. A snapshot exists only while a +trace records one, so an aggregate scan that runs without a trace has nothing to +compare the key against. Still apply every check that does not need the snapshot. +""" +function _nestedwritekeyassert!(::Nothing, value, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits, trace, witness, + row_witness) + _nestedwriterowcheck(row_witness) + _nestedwritekeywitnessroot(witness, value) + _nestedwritekeydeclared(shape, value) + return +end + +function _nestedwritekeyassert!(expected::_NestedWriteKeySnapshot, value, + shape::Union{Nothing,_NestedWriteShape}, limits::Limits, + trace::_NestedWriteTrace, witness, row_witness) + _nestedwriterowcheck(row_witness) + _nestedwritekeywitnessroot(witness, value) + _nestedwritekeydeclared(shape, value) + _nestedwritekeyequal(expected, value, trace, limits, shape, witness, + row_witness) || throw(ArgumentError( + "nested Parquet MAP key changed while it was consumed")) + return +end + +function _nestedwriteoccurrencepayload(value) + value isa AbstractString && return Int64(ncodeunits(value)) + value isa AbstractVector{UInt8} && return Int64(length(value)) + value isa JSONValue && return Int64(length(value.bytes)) + value isa BSONValue && return Int64(length(value.bytes)) + value isa UUIDs.UUID && return Int64(16) + value isa Interval && return Int64(12) + return Int64(0) +end + +function _nestedwriteoccurrencelogical(value::Decimal) + return Int64(value.scale), Int64(_decimaldigits(value.unscaled)) +end + +function _nestedwriteoccurrencelogical(value::Timestamp) + adjusted = value.is_adjusted_to_utc ? Int64(1) : Int64(0) + return adjusted, Int64(0) +end + +function _nestedwriteoccurrencelogical(value) + return Int64(0), Int64(0) +end + +function _nestedwritetraceleaf!(trace, value, present::Bool) + payload = present ? _nestedwriteoccurrencepayload(value) : Int64(0) + logical, detail = present ? _nestedwriteoccurrencelogical(value) : + (Int64(0), Int64(0)) + _nestedwritetraceevent!(trace, _NESTED_WRITE_TRACE_LEAF, + present ? Int64(1) : Int64(0), payload, logical, detail) + return +end + +function _nestedwritetracestruct!(trace, value, present::Bool, count::Int) + _nestedwritetraceevent!(trace, _NESTED_WRITE_TRACE_STRUCT, + present ? Int64(1) : Int64(0), Int64(count), Int64(0), Int64(0)) + return +end + +function _nestedwritetracecontainer!(trace, kind::UInt8, value, + present::Bool) + if !present + _nestedwritetraceevent!(trace, kind, Int64(0), Int64(0), Int64(0), + Int64(0)) + return + end + value isa ListValue && _validatelistvalue(value) + value isa MapValue && _validatemapvalue(value) + count = value isa AbstractVector ? + Int64(_nestedvectorcount(value, "nested Parquet container")) : + Int64(_nestedcheckedint(length(value), + "nested Parquet container length")) + if value isa AbstractVector + rawfirst, rawlast = _nestedvectoraxes(value, + "nested Parquet container") + first = Int64(rawfirst) + last = Int64(rawlast) + else + first = typemin(Int64) + last = typemin(Int64) + end + _nestedwritetraceevent!(trace, kind, Int64(1), count, first, last) + return +end + +function _nestedwritetracedict!(trace, count::Int) + _nestedwritetraceevent!(trace, _NESTED_WRITE_TRACE_MAP, Int64(1), + Int64(count), typemin(Int64), typemin(Int64)) + return +end + +function _nestedwritesnapshotarraycharge(values::BitVector) + return _materializedbitbytes(length(values)) +end + +function _nestedwritesnapshotarraycharge(values::AbstractVector) + return _materializedarraybytes(eltype(values), length(values)) +end + +function _nestedwritevalidatesource(values::AbstractVector, limits::Limits, + budget::_LiveByteBudget; depth::Int=1) + Base.@nospecialize values + stack = _nestedwritepassstackstart(_NestedWriteSourceFrame, budget) + try + _nestedwritestackpush!(stack, + _NestedWriteSourceFrame(values, depth, 0, -1, false), budget) + while !isempty(stack) + frame = _nestedwritepassstackpop!(stack) + try + if frame.position == 0 + _checklimit(:metadata_depth, frame.depth, + limits.max_metadata_depth) + _nestedwritevalidatesourcenode(frame.values) + frame = _NestedWriteSourceFrame(frame.values, frame.depth, + 0, _nestedwritesourcechildcount(frame.values), false) + end + _nestedwritesourceschedule!(stack, frame, limits, budget) + finally + _nestedwritepassstackprocessed!(stack) + end + end + finally + _nestedwritepassstackrelease!(stack, budget) + end + return +end + +function _nestedwritesnapshotmetrics(values::AbstractVector, limits::Limits, + budget::_LiveByteBudget; depth::Int=1) + Base.@nospecialize values + stack = _nestedwritepassstackstart(_NestedWriteSourceFrame, budget) + count = 0 + charge = Int64(0) + try + _nestedwritestackpush!(stack, + _NestedWriteSourceFrame(values, depth, 0, -1, false), budget) + while !isempty(stack) + frame = _nestedwritepassstackpop!(stack) + try + if frame.position == 0 + _checklimit(:metadata_depth, frame.depth, + limits.max_metadata_depth) + count = Base.checked_add(count, 1) + charge = _materializedsum(charge, + _nestedwritesnapshotcopycharge(frame.values)) + frame = _NestedWriteSourceFrame(frame.values, frame.depth, + 0, _nestedwritesourcechildcount(frame.values), false) + end + _nestedwritesourceschedule!(stack, frame, limits, budget) + finally + _nestedwritepassstackprocessed!(stack) + end + end + finally + _nestedwritepassstackrelease!(stack, budget) + end + return count, charge +end + +function _nestedwritesnapshotnode!(nodes::Vector{_NestedWriteVectorSnapshot}, + values::AbstractVector, limits::Limits, budget::_LiveByteBudget; + depth::Int=1) + Base.@nospecialize values + stack = _nestedwritepassstackstart(_NestedWriteSourceFrame, budget) + try + _nestedwritestackpush!(stack, + _NestedWriteSourceFrame(values, depth, 0, -1, false), budget) + while !isempty(stack) + frame = _nestedwritepassstackpop!(stack) + try + if frame.position == 0 + _checklimit(:metadata_depth, frame.depth, + limits.max_metadata_depth) + _nestedwritesnapshotappend!(nodes, frame.values) + frame = _NestedWriteSourceFrame(frame.values, frame.depth, + 0, _nestedwritesourcechildcount(frame.values), false) + end + _nestedwritesourceschedule!(stack, frame, limits, budget) + finally + _nestedwritepassstackprocessed!(stack) + end + end + finally + _nestedwritepassstackrelease!(stack, budget) + end + return +end + +function _nestedwritesnapshotlookupcapacity(count::Int) + count >= 0 || throw(ArgumentError( + "nested Parquet snapshot count must be nonnegative")) + required = try + Base.checked_mul(count, 2) + catch error + error isa OverflowError || rethrow() + throw(LimitError(:materialized_bytes, typemax(Int64), typemax(Int64))) + end + capacity = 4 + while capacity < required + capacity = try + Base.checked_mul(capacity, 2) + catch error + error isa OverflowError || rethrow() + throw(LimitError(:materialized_bytes, typemax(Int64), + typemax(Int64))) + end + end + return capacity +end + +function _nestedwritesnapshotlookuparraycharge(capacity::Int) + charge = _materializedarraybytes(Union{Nothing,AbstractVector}, capacity) + return _materializedsum(charge, _materializedarraybytes( + Union{Nothing,_NestedWriteVectorSnapshot}, capacity)) +end + +function _nestedwritesnapshotlookupcharge(capacity::Int) + return _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _nestedwritesnapshotlookuparraycharge(capacity)) +end + +function _nestedwritesnapshotlookup(count::Int) + capacity = _nestedwritesnapshotlookupcapacity(count) + sources = Vector{Union{Nothing,AbstractVector}}(undef, capacity) + snapshots = Vector{Union{Nothing,_NestedWriteVectorSnapshot}}( + undef, capacity) + fill!(sources, nothing) + fill!(snapshots, nothing) + return _NestedWriteSnapshotLookup(sources, snapshots, 0, + _nestedwritesnapshotlookupcharge(capacity)) +end + +function _nestedwritesnapshotlookupslot( + lookup::_NestedWriteSnapshotLookup, source::AbstractVector) + capacity = length(lookup.sources) + mask = UInt(capacity - 1) + index = Int((objectid(source) & mask) + UInt(1)) + while true + stored = lookup.sources[index] + (stored === nothing || stored === source) && return index + index = index == capacity ? 1 : index + 1 + end +end + +function _nestedwritesnapshotlookupget( + lookup::_NestedWriteSnapshotLookup, source::AbstractVector) + index = _nestedwritesnapshotlookupslot(lookup, source) + lookup.sources[index] === source || return nothing + return lookup.snapshots[index] +end + +function _nestedwritesnapshotlookupinsertarrays!( + sources::Vector{Union{Nothing,AbstractVector}}, + snapshots::Vector{Union{Nothing,_NestedWriteVectorSnapshot}}, + source::AbstractVector, snapshot::_NestedWriteVectorSnapshot) + capacity = length(sources) + mask = UInt(capacity - 1) + index = Int((objectid(source) & mask) + UInt(1)) + while sources[index] !== nothing + sources[index] === source && return false + index = index == capacity ? 1 : index + 1 + end + sources[index] = source + snapshots[index] = snapshot + return true +end + +function _nestedwritesnapshotlookupinsert!( + lookup::_NestedWriteSnapshotLookup, source::AbstractVector, + snapshot::_NestedWriteVectorSnapshot) + _nestedwritesnapshotlookupinsertarrays!(lookup.sources, lookup.snapshots, + source, snapshot) || return false + lookup.count = Base.checked_add(lookup.count, 1) + return true +end + +function _nestedwritesnapshotlookupgrow!( + topology::_NestedWriteTopologySnapshot, budget::_LiveByteBudget) + lookup = topology.lookup + newcapacity = _nestedwritesnapshotlookupcapacity( + Base.checked_add(lookup.count, 1)) + newcapacity <= length(lookup.sources) && return + newcharge = _nestedwritesnapshotlookuparraycharge(newcapacity) + _reserve!(budget, newcharge) + sources = nothing + snapshots = nothing + try + sources = Vector{Union{Nothing,AbstractVector}}(undef, newcapacity) + snapshots = Vector{Union{Nothing,_NestedWriteVectorSnapshot}}( + undef, newcapacity) + fill!(sources, nothing) + fill!(snapshots, nothing) + catch + _release!(budget, newcharge) + rethrow() + end + count = 0 + for index in eachindex(lookup.sources) + source = lookup.sources[index] + source === nothing && continue + _nestedwritesnapshotlookupinsertarrays!(sources, snapshots, source, + something(lookup.snapshots[index])) || throw(AssertionError( + "nested Parquet snapshot lookup contains duplicate identities")) + count = Base.checked_add(count, 1) + end + oldcharge = _nestedwritesnapshotlookuparraycharge(length(lookup.sources)) + topologycharge = _materializedsum(topology.charge, newcharge) + topologycharge = Base.checked_sub(topologycharge, oldcharge) + lookupcharge = Base.checked_add(Base.checked_sub(lookup.charge, + oldcharge), newcharge) + lookup.sources = sources + lookup.snapshots = snapshots + lookup.count = count + lookup.charge = lookupcharge + topology.charge = topologycharge + _release!(budget, oldcharge) + return +end + +function _nestedwritesnapshotcopycharge(values::AbstractVector) + charge = Int64(0) + if values isa ListVector + charge = _nestedwritesnapshotarraycharge(values.offsets) + values.validity === nothing || (charge = _materializedsum(charge, + _nestedwritesnapshotarraycharge(values.validity))) + elseif values isa StructVector + charge = _nestedwritesnapshotarraycharge(values.names) + charge = _materializedsum(charge, + _nestedwritesnapshotarraycharge(values.children)) + values.ranks === nothing || (charge = _materializedsum(charge, + _nestedwritesnapshotarraycharge(values.ranks))) + elseif values isa MapVector + charge = _nestedwritesnapshotarraycharge(values.offsets) + values.validity === nothing || (charge = _materializedsum(charge, + _nestedwritesnapshotarraycharge(values.validity))) + end + return charge +end + +function _nestedwritevalidatesourcenode(values::AbstractVector) + _nestedvectorcount(values, "nested Parquet vector") + _nestedvectoraxes(values, "nested Parquet vector") + values isa ListVector && _validatelistvector(values) + values isa StructVector && _validatestructvector(values) + values isa MapVector && _validatemapvector(values) + return +end + +function _nestedwritesnapshotappend!(nodes::Vector{_NestedWriteVectorSnapshot}, + values::AbstractVector) + count = _nestedvectorcount(values, "nested Parquet vector") + first, last = _nestedvectoraxes(values, "nested Parquet vector") + if values isa LogicalColumn + push!(nodes, _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_LOGICAL, count, first, last, values.values, + values.spec, nothing, nothing, nothing, nothing, nothing, Int64(0))) + elseif values isa FixedByteArrayVector + push!(nodes, _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_FIXED, count, first, last, values.values, + nothing, nothing, nothing, nothing, nothing, nothing, + Int64(values.width))) + elseif values isa ListVector + offsets = copy(values.offsets) + validity = values.validity === nothing ? nothing : copy(values.validity) + push!(nodes, _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_LIST, count, first, last, values.offsets, + values.validity, values.values, nothing, offsets, validity, nothing, + Int64(0))) + elseif values isa StructVector + names = copy(values.names) + ranks = values.ranks === nothing ? nothing : copy(values.ranks) + children = copy(values.children) + push!(nodes, _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_STRUCT, count, first, last, values.names, + values.ranks, values.children, nothing, names, ranks, children, + Int64(values.rows))) + elseif values isa MapVector + offsets = copy(values.offsets) + validity = values.validity === nothing ? nothing : copy(values.validity) + push!(nodes, _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_MAP, count, first, last, values.offsets, + values.validity, values.keys, values.values, offsets, validity, + nothing, Int64(0))) + else + push!(nodes, _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_GENERIC, count, first, last, nothing, nothing, + nothing, nothing, nothing, nothing, nothing, Int64(0))) + end + return nodes[end] +end + +function _nestedwritesnapshotappendlocal!( + nodes::Vector{_NestedWriteVectorSnapshot}, values::ListVector) + _validatelistlocal(values) + count = length(values.offsets) - 1 + offsets = copy(values.offsets) + validity = values.validity === nothing ? nothing : copy(values.validity) + snapshot = _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_LIST, count, 1, count, values.offsets, + values.validity, values.values, nothing, offsets, validity, nothing, + Int64(0)) + push!(nodes, snapshot) + _nestedwritevalidatenodelocal(snapshot) + return snapshot +end + +function _nestedwritesnapshotappendlocal!( + nodes::Vector{_NestedWriteVectorSnapshot}, values::StructVector) + _validatestructlocal(values) + count = values.rows + names = copy(values.names) + ranks = values.ranks === nothing ? nothing : copy(values.ranks) + children = copy(values.children) + snapshot = _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_STRUCT, count, 1, count, values.names, + values.ranks, values.children, nothing, names, ranks, children, + Int64(values.rows)) + push!(nodes, snapshot) + _nestedwritevalidatenodelocal(snapshot) + return snapshot +end + +function _nestedwritesnapshotappendlocal!( + nodes::Vector{_NestedWriteVectorSnapshot}, values::MapVector) + _validatemaplocal(values) + count = length(values.offsets) - 1 + offsets = copy(values.offsets) + validity = values.validity === nothing ? nothing : copy(values.validity) + snapshot = _NestedWriteVectorSnapshot(values, + _NESTED_WRITE_VECTOR_MAP, count, 1, count, values.offsets, + values.validity, values.keys, values.values, offsets, validity, + nothing, Int64(0)) + push!(nodes, snapshot) + _nestedwritevalidatenodelocal(snapshot) + return snapshot +end + +function _nestedwriteextendtopologylocal!( + topology::_NestedWriteTopologySnapshot, values::AbstractVector, + budget::_LiveByteBudget) + existing = _nestedwritesnapshotlookupget(topology.lookup, values) + if existing !== nothing + _nestedwritevalidatenodelocal(existing) + return existing + end + _nestedwritesnapshotlookupgrow!(topology, budget) + oldcount = length(topology.nodes) + newcount = Base.checked_add(oldcount, 1) + arraycharge = Base.checked_sub( + _materializedarraybytes(_NestedWriteVectorSnapshot, newcount), + _materializedarraybytes(_NestedWriteVectorSnapshot, oldcount)) + charge = _materializedsum(arraycharge, _MATERIALIZED_OBJECT_BYTES) + charge = _materializedsum(charge, + _nestedwritesnapshotcopycharge(values)) + _reserve!(budget, charge) + topology.charge = _materializedsum(topology.charge, charge) + sizehint!(topology.nodes, newcount) + snapshot = if values isa Union{ListVector,StructVector,MapVector} + _nestedwritesnapshotappendlocal!(topology.nodes, values) + else + _nestedwritesnapshotappend!(topology.nodes, values) + end + _nestedwritesnapshotlookupinsert!(topology.lookup, values, snapshot) || + throw(AssertionError( + "nested Parquet topology inserted a duplicate local snapshot")) + return snapshot +end + +function _nestedwriteextendtopology!(topology::_NestedWriteTopologySnapshot, + values::AbstractVector, limits::Limits, budget::_LiveByteBudget, + depth::Int=1) + Base.@nospecialize values + stack = _nestedwritepassstackstart(_NestedWriteSourceFrame, budget) + root::Union{Nothing,_NestedWriteVectorSnapshot} = nothing + try + _nestedwritestackpush!(stack, + _NestedWriteSourceFrame(values, depth, 0, -1, false), budget) + while !isempty(stack) + frame = _nestedwritepassstackpop!(stack) + try + if frame.exit + snapshot = _nestedwritesnapshotlookupget(topology.lookup, + frame.values) + snapshot === nothing && throw(AssertionError( + "nested Parquet topology lost a source snapshot")) + _nestedwritevalidatenodelocal(snapshot) + continue + end + if frame.position == 0 + _checklimit(:metadata_depth, frame.depth, + limits.max_metadata_depth) + existing = _nestedwritesnapshotlookupget(topology.lookup, + frame.values) + if existing !== nothing + root === nothing && (root = existing) + continue + end + _nestedwritesnapshotlookupgrow!(topology, budget) + oldcount = length(topology.nodes) + newcount = Base.checked_add(oldcount, 1) + arraycharge = Base.checked_sub( + _materializedarraybytes(_NestedWriteVectorSnapshot, + newcount), + _materializedarraybytes(_NestedWriteVectorSnapshot, + oldcount)) + charge = _materializedsum(arraycharge, + _MATERIALIZED_OBJECT_BYTES) + charge = _materializedsum(charge, + _nestedwritesnapshotcopycharge(frame.values)) + _reserve!(budget, charge) + topology.charge = _materializedsum(topology.charge, charge) + sizehint!(topology.nodes, newcount) + snapshot = _nestedwritesnapshotappend!(topology.nodes, + frame.values) + _nestedwritesnapshotlookupinsert!(topology.lookup, + frame.values, snapshot) || throw(AssertionError( + "nested Parquet topology inserted a duplicate source snapshot")) + root === nothing && (root = snapshot) + _nestedwritevalidatenodelocal(snapshot) + _nestedwritevalidatesourcenode(frame.values) + _nestedwritevalidatenodelocal(snapshot) + frame = _NestedWriteSourceFrame(frame.values, frame.depth, + 0, _nestedwritesourcechildcount(frame.values), false) + if iszero(frame.count) + _nestedwritevalidatenodelocal(snapshot) + continue + end + end + _nestedwritesourceschedule!(stack, frame, limits, budget; + postorder=true) || throw(AssertionError( + "nested Parquet topology source traversal stopped early")) + finally + _nestedwritepassstackprocessed!(stack) + end + end + finally + _nestedwritepassstackrelease!(stack, budget) + end + root === nothing && throw(AssertionError( + "nested Parquet topology did not produce a root snapshot")) + return root +end + +function _nestedwritetraceviewsnapshot!(::Nothing, source::AbstractVector, + snapshot, ::Limits) + snapshot === nothing || snapshot.source === source || throw(ArgumentError( + "nested Parquet view changed its backing source")) + return snapshot +end + +function _nestedwritetraceviewsnapshot!(trace::_NestedWriteTrace, + source::AbstractVector, snapshot, limits::Limits) + if trace.capturing + topology = trace.topology + topology isa _NestedWriteTopologySnapshot || throw(AssertionError( + "nested Parquet trace has no topology authority")) + stored = snapshot === nothing ? _nestedwriteextendtopology!(topology, + source, limits, trace.budget) : snapshot + stored.source === source || throw(ArgumentError( + "nested Parquet view changed its backing source")) + _nestedwritetracecapture!(trace, _NestedWriteTraceEvent( + _NESTED_WRITE_TRACE_SOURCE, Int64(0), Int64(0), Int64(0), + Int64(0), stored)) + return stored + end + event = _nestedwritetracenext!(trace) + event.kind == _NESTED_WRITE_TRACE_SOURCE && iszero(event.a) && + iszero(event.b) && iszero(event.c) && iszero(event.d) || throw( + ArgumentError( + "nested Parquet input changed its view topology between writer passes")) + stored = event.value::_NestedWriteVectorSnapshot + stored.source === source || throw(ArgumentError( + "nested Parquet view changed its backing source between writer passes")) + snapshot === nothing || snapshot === stored || throw(ArgumentError( + "nested Parquet view changed its topology authority")) + return stored +end + +function _nestedwritetopology(input_columns::Union{Nothing,AbstractVector}, + names::Vector{String}, values::Vector{AbstractVector}, limits::Limits, + budget::_LiveByteBudget; retainedcharge::Int64=Int64(0)) + start = _budgetused(budget) + try + nodecount = 0 + copycharge = Int64(0) + for value in values + _nestedwritevalidatesource(value, limits, budget) + count, charge = _nestedwritesnapshotmetrics(value, limits, budget) + nodecount = Base.checked_add(nodecount, count) + copycharge = _materializedsum(copycharge, charge) + end + localcharge = _materializedsum( + _materializedarraybytes(_NestedWriteVectorSnapshot, nodecount), + _materializedproduct(nodecount + 1, + _MATERIALIZED_OBJECT_BYTES)) + lookupcapacity = _nestedwritesnapshotlookupcapacity(nodecount) + localcharge = _materializedsum(localcharge, + _nestedwritesnapshotlookupcharge(lookupcapacity)) + localcharge = _materializedsum(localcharge, copycharge) + _reserve!(budget, localcharge) + charge = _materializedsum(localcharge, retainedcharge) + nodes = _NestedWriteVectorSnapshot[] + sizehint!(nodes, nodecount) + for value in values + _nestedwritesnapshotnode!(nodes, value, limits, budget) + end + length(nodes) == nodecount || throw(ArgumentError( + "nested Parquet vector topology changed while it was copied")) + lookup = _nestedwritesnapshotlookup(nodecount) + for snapshot in nodes + _nestedwritesnapshotlookupinsert!(lookup, snapshot.source, + snapshot) + end + return _NestedWriteTopologySnapshot(input_columns, + input_columns === nothing ? 0 : length(input_columns), + input_columns === nothing ? 1 : firstindex(input_columns), + input_columns === nothing ? 0 : lastindex(input_columns), names, + values, nodes, lookup, nodecount, charge) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _nestedwritesnapshotfor(topology::_NestedWriteTopologySnapshot, + source::AbstractVector) + snapshot = _nestedwritesnapshotlookupget(topology.lookup, source) + snapshot === nothing && throw(ArgumentError( + "nested Parquet source vector is absent from its topology snapshot")) + return snapshot +end + +function _nestedwriterowcheck(witness::Nothing) + return +end + +function _nestedwriterowcheck(witness::_NestedWriteRowWitness) + snapshot = witness.snapshot + values = snapshot.source + index = witness.index + kind = snapshot.kind + _nestedvectorcount(values, "nested Parquet row source") == snapshot.count || + throw(ArgumentError( + "nested Parquet vector length changed during row access")) + firstaxis, lastaxis = _nestedvectoraxes(values, + "nested Parquet row source") + firstaxis == snapshot.first && lastaxis == snapshot.last || throw( + ArgumentError( + "nested Parquet vector axes changed during row access")) + if kind == _NESTED_WRITE_VECTOR_LIST + values isa ListVector || throw(ArgumentError( + "nested Parquet LIST wrapper changed during row access")) + values.offsets === snapshot.primary && + values.validity === snapshot.secondary && + values.values === snapshot.tertiary || throw(ArgumentError( + "nested Parquet LIST backing topology changed during row access")) + offsets = values.offsets + length(offsets) == length(snapshot.copy1) || throw(ArgumentError( + "nested Parquet LIST offset count changed during row access")) + offsets[index] == snapshot.copy1[index] && + offsets[index + 1] == snapshot.copy1[index + 1] || throw( + ArgumentError( + "nested Parquet LIST offsets changed during row access")) + validity = values.validity + validity === nothing || length(validity) == length(snapshot.copy2) || + throw(ArgumentError( + "nested Parquet LIST validity count changed during row access")) + validity === nothing || validity[index] == snapshot.copy2[index] || + throw(ArgumentError( + "nested Parquet LIST validity changed during row access")) + elseif kind == _NESTED_WRITE_VECTOR_STRUCT + values isa StructVector || throw(ArgumentError( + "nested Parquet struct wrapper changed during row access")) + values.rows == snapshot.scalar && values.names === snapshot.primary && + values.ranks === snapshot.secondary && + values.children === snapshot.tertiary || throw(ArgumentError( + "nested Parquet struct backing topology changed during row access")) + ranks = values.ranks + ranks === nothing || length(ranks) == length(snapshot.copy2) || throw( + ArgumentError( + "nested Parquet struct rank count changed during row access")) + ranks === nothing || (ranks[index] == snapshot.copy2[index] && + ranks[index + 1] == snapshot.copy2[index + 1]) || throw( + ArgumentError( + "nested Parquet struct ranks changed during row access")) + elseif kind == _NESTED_WRITE_VECTOR_MAP + values isa MapVector || throw(ArgumentError( + "nested Parquet MAP wrapper changed during row access")) + values.offsets === snapshot.primary && + values.validity === snapshot.secondary && + values.keys === snapshot.tertiary && + values.values === snapshot.quaternary || throw(ArgumentError( + "nested Parquet MAP backing topology changed during row access")) + offsets = values.offsets + length(offsets) == length(snapshot.copy1) || throw(ArgumentError( + "nested Parquet MAP offset count changed during row access")) + offsets[index] == snapshot.copy1[index] && + offsets[index + 1] == snapshot.copy1[index + 1] || throw( + ArgumentError( + "nested Parquet MAP offsets changed during row access")) + validity = values.validity + validity === nothing || length(validity) == length(snapshot.copy2) || + throw(ArgumentError( + "nested Parquet MAP validity count changed during row access")) + validity === nothing || validity[index] == snapshot.copy2[index] || + throw(ArgumentError( + "nested Parquet MAP validity changed during row access")) + elseif kind == _NESTED_WRITE_VECTOR_LOGICAL + values isa LogicalColumn && values.values === snapshot.primary && + isequal(values.spec, snapshot.secondary) || throw(ArgumentError( + "nested Parquet logical wrapper changed during row access")) + elseif kind == _NESTED_WRITE_VECTOR_FIXED + values isa FixedByteArrayVector && values.values === snapshot.primary && + values.width == snapshot.scalar || throw(ArgumentError( + "nested Parquet fixed-width wrapper changed during row access")) + elseif kind == _NESTED_WRITE_VECTOR_GENERIC + nothing + else + throw(AssertionError("row witness has a non-container snapshot")) + end + return +end + +function _nestedwriterowwitness(snapshot::Nothing, ::Int) + return nothing +end + +function _nestedwriterowwitness(snapshot::_NestedWriteVectorSnapshot, + index::Int) + kind = snapshot.kind + if kind in (_NESTED_WRITE_VECTOR_LIST, _NESTED_WRITE_VECTOR_STRUCT, + _NESTED_WRITE_VECTOR_MAP) + 1 <= index <= snapshot.count || throw(ArgumentError( + "nested Parquet row index exceeds its topology snapshot")) + else + checkbounds(Bool, snapshot.source, index) || throw(ArgumentError( + "nested Parquet row index exceeds its topology snapshot")) + end + if kind == _NESTED_WRITE_VECTOR_STRUCT + ranks = snapshot.copy2 + if ranks === nothing + witness = _NestedWriteRowWitness(snapshot, index, index, index, + true) + else + first = Int(ranks[index]) + last = Int(ranks[index + 1]) + witness = _NestedWriteRowWitness(snapshot, index, first, last, + first != last) + end + elseif kind in (_NESTED_WRITE_VECTOR_LIST, _NESTED_WRITE_VECTOR_MAP) + offsets = snapshot.copy1 + first, last = _nestedspan(offsets, index) + validity = snapshot.copy2 + present = validity === nothing || validity[index] + witness = _NestedWriteRowWitness(snapshot, index, first, last, present) + else + witness = _NestedWriteRowWitness(snapshot, index, index, index, true) + end + _nestedwriterowcheck(witness) + return witness +end + +function _nestedwriterowvalue(values::AbstractVector, index::Int, + ::Nothing) + return values[index] +end + +function _nestedwriterowvalue(::AbstractVector, ::Int, + witness::_NestedWriteRowWitness) + snapshot = witness.snapshot + witness.present || return missing + if snapshot.kind == _NESTED_WRITE_VECTOR_LIST + return ListValue(snapshot.tertiary, witness.first, witness.last) + elseif snapshot.kind == _NESTED_WRITE_VECTOR_STRUCT + return StructValue(snapshot.primary, snapshot.tertiary, witness.last) + elseif snapshot.kind == _NESTED_WRITE_VECTOR_MAP + keys = snapshot.tertiary + values = snapshot.quaternary + K = eltype(keys) + hasvalues = values !== nothing + V = hasvalues ? eltype(values) : Missing + return MapValue{K,V,hasvalues}(keys, values, witness.first, + witness.last) + end + return snapshot.source[witness.index] +end + +function _nestedwriterowaccess(values::AbstractVector, index::Int, snapshot) + witness = _nestedwriterowwitness(snapshot, index) + value = _nestedwriterowvalue(values, index, witness) + _nestedwriterowcheck(witness) + return value, witness +end + +function _nestedwritearrayequal(current::AbstractVector, + expected::AbstractVector) + typeof(current) === typeof(expected) || return false + length(current) == length(expected) || return false + firstindex(current) == firstindex(expected) && + lastindex(current) == lastindex(expected) || return false + for index in eachindex(current, expected) + isequal(current[index], expected[index]) || return false + end + return true +end + +function _nestedwritevectoridentities(current::Vector{AbstractVector}, + expected::Vector{AbstractVector}) + length(current) == length(expected) || return false + for index in eachindex(current, expected) + current[index] === expected[index] || return false + end + return true +end + +function _nestedwritevalidateaxis(snapshot::_NestedWriteVectorSnapshot) + values = snapshot.source + _nestedvectorcount(values, "nested Parquet vector") == snapshot.count || + throw(ArgumentError( + "nested Parquet vector length changed between writer phases")) + first, last = _nestedvectoraxes(values, "nested Parquet vector") + first == snapshot.first && last == snapshot.last || throw(ArgumentError( + "nested Parquet vector axes changed between writer phases")) + return +end + +function _nestedwritevalidatenodelocal(snapshot::_NestedWriteVectorSnapshot) + values = snapshot.source + kind = snapshot.kind + if kind == _NESTED_WRITE_VECTOR_LOGICAL + values isa LogicalColumn || throw(ArgumentError( + "nested Parquet logical wrapper changed between writer phases")) + values.values === snapshot.primary || throw(ArgumentError( + "nested Parquet logical wrapper changed child identity")) + isequal(values.spec, snapshot.secondary) || throw(ArgumentError( + "nested Parquet logical metadata changed between writer phases")) + elseif kind == _NESTED_WRITE_VECTOR_FIXED + values isa FixedByteArrayVector || throw(ArgumentError( + "nested Parquet fixed-width wrapper changed between writer phases")) + values.values === snapshot.primary || throw(ArgumentError( + "nested Parquet fixed-width wrapper changed child identity")) + values.width == snapshot.scalar || throw(ArgumentError( + "nested Parquet fixed width changed between writer phases")) + elseif kind == _NESTED_WRITE_VECTOR_LIST + values isa ListVector || throw(ArgumentError( + "nested Parquet LIST wrapper changed between writer phases")) + values.offsets === snapshot.primary && + _nestedwritearrayequal(values.offsets, snapshot.copy1) || + throw(ArgumentError( + "nested Parquet LIST offsets changed between writer phases")) + values.validity === snapshot.secondary || throw(ArgumentError( + "nested Parquet LIST validity identity changed between writer phases")) + values.validity === nothing || + _nestedwritearrayequal(values.validity, snapshot.copy2) || + throw(ArgumentError( + "nested Parquet LIST validity changed between writer phases")) + values.values === snapshot.tertiary || throw(ArgumentError( + "nested Parquet LIST child identity changed between writer phases")) + elseif kind == _NESTED_WRITE_VECTOR_STRUCT + values isa StructVector || throw(ArgumentError( + "nested Parquet struct wrapper changed between writer phases")) + values.rows == snapshot.scalar || throw(ArgumentError( + "nested Parquet struct row count changed between writer phases")) + values.names === snapshot.primary && + _nestedwritearrayequal(values.names, snapshot.copy1) || + throw(ArgumentError( + "nested Parquet struct names changed between writer phases")) + values.ranks === snapshot.secondary || throw(ArgumentError( + "nested Parquet struct rank identity changed between writer phases")) + values.ranks === nothing || + _nestedwritearrayequal(values.ranks, snapshot.copy2) || + throw(ArgumentError( + "nested Parquet struct ranks changed between writer phases")) + values.children === snapshot.tertiary && + _nestedwritevectoridentities(values.children, snapshot.copy3) || + throw(ArgumentError( + "nested Parquet struct child identity or order changed between writer phases")) + elseif kind == _NESTED_WRITE_VECTOR_MAP + values isa MapVector || throw(ArgumentError( + "nested Parquet MAP wrapper changed between writer phases")) + values.offsets === snapshot.primary && + _nestedwritearrayequal(values.offsets, snapshot.copy1) || + throw(ArgumentError( + "nested Parquet MAP offsets changed between writer phases")) + values.validity === snapshot.secondary || throw(ArgumentError( + "nested Parquet MAP validity identity changed between writer phases")) + values.validity === nothing || + _nestedwritearrayequal(values.validity, snapshot.copy2) || + throw(ArgumentError( + "nested Parquet MAP validity changed between writer phases")) + values.keys === snapshot.tertiary || throw(ArgumentError( + "nested Parquet MAP key identity changed between writer phases")) + values.values === snapshot.quaternary || throw(ArgumentError( + "nested Parquet MAP value identity changed between writer phases")) + elseif kind != _NESTED_WRITE_VECTOR_GENERIC + throw(AssertionError("unknown nested writer vector snapshot kind")) + end + return +end + +function _nestedwritevalidatenode(snapshot::_NestedWriteVectorSnapshot) + _nestedwritevalidatenodelocal(snapshot) + values = snapshot.source + kind = snapshot.kind + kind == _NESTED_WRITE_VECTOR_LIST && _validatelistvector(values) + kind == _NESTED_WRITE_VECTOR_STRUCT && _validatestructvector(values) + kind == _NESTED_WRITE_VECTOR_MAP && _validatemapvector(values) + _nestedwritevalidateaxis(snapshot) + _nestedwritevalidatenodelocal(snapshot) + return +end + +function _nestedwritedictreserve!(::Nothing, + materialization::_NestedWriteDictMaterialization, ::Int64) + return +end + +function _nestedwritedictreserve!(trace::_NestedWriteTrace, + materialization::_NestedWriteDictMaterialization, bytes::Int64) + _nestedwritetracereserve!(trace, bytes) + materialization.charge = _materializedsum(materialization.charge, bytes) + return +end + +function _nestedwritedictunreserve!(::Nothing, + materialization::_NestedWriteDictMaterialization, ::Int64) + return +end + +function _nestedwritedictunreserve!(trace::_NestedWriteTrace, + materialization::_NestedWriteDictMaterialization, bytes::Int64) + _nestedwritetraceunreserve!(trace, bytes) + materialization.charge = Base.checked_sub(materialization.charge, bytes) + return +end + +function _nestedwritedictmaterialization(trace) + capacity = _nestedwritesnapshotlookupcapacity(0) + setcharge = _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _materializedarraybytes(Union{Nothing,AbstractVector}, capacity)) + charge = _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _materializedproduct(2, setcharge)) + trace isa _NestedWriteTrace && _nestedwritetracereserve!(trace, charge) + try + sources = Vector{Union{Nothing,AbstractVector}}(undef, capacity) + fill!(sources, nothing) + dependencies = _NestedWriteIdentitySet(sources, 0) + preflight_sources = Vector{Union{Nothing,AbstractVector}}(undef, + capacity) + fill!(preflight_sources, nothing) + preflight = _NestedWriteIdentitySet(preflight_sources, 0) + return _NestedWriteDictMaterialization(nothing, nothing, nothing, + nothing, dependencies, preflight, 0, + trace isa _NestedWriteTrace ? charge : Int64(0)) + catch + trace isa _NestedWriteTrace && _nestedwritetraceunreserve!(trace, + charge) + rethrow() + end +end + +function _nestedwritedictrelease!(trace, + materialization::_NestedWriteDictMaterialization) + entry = materialization.first + while entry !== nothing + following = entry.next + entry.key = nothing + entry.value = nothing + entry.next = nothing + entry = following + end + dependency = materialization.dependencies + while dependency !== nothing + following = dependency.next + dependency.next = nothing + dependency = following + end + materialization.first = nothing + materialization.last = nothing + materialization.dependencies = nothing + materialization.dependency_last = nothing + materialization.dependency_sources = nothing + materialization.preflight_sources = nothing + materialization.count = 0 + charge = materialization.charge + if !iszero(charge) && trace isa _NestedWriteTrace + _nestedwritetraceunreserve!(trace, charge) + end + materialization.charge = Int64(0) + return +end + +function _nestedwritedictentry!(trace, + materialization::_NestedWriteDictMaterialization, pair::Pair) + _nestedwritedictreserve!(trace, materialization, + _MATERIALIZED_OBJECT_BYTES) + entry = try + _NestedWriteDictEntry(pair.first, pair.second, nothing) + catch + _nestedwritedictunreserve!(trace, materialization, + _MATERIALIZED_OBJECT_BYTES) + rethrow() + end + if materialization.last === nothing + materialization.first = entry + else + materialization.last.next = entry + end + materialization.last = entry + return +end + +function _nestedwritedictidentityslot( + sources::Vector{Union{Nothing,AbstractVector}}, + source::AbstractVector) + capacity = length(sources) + mask = UInt(capacity - 1) + index = Int((objectid(source) & mask) + UInt(1)) + while true + stored = sources[index] + (stored === nothing || stored === source) && return index + index = index == capacity ? 1 : index + 1 + end +end + +function _nestedwritedictidentitygrow!(trace, + materialization::_NestedWriteDictMaterialization, + identities::_NestedWriteIdentitySet) + capacity = length(identities.sources) + Base.checked_mul(Base.checked_add(identities.count, 1), 2) <= capacity && + return + newcapacity = Base.checked_mul(capacity, 2) + newcharge = _materializedarraybytes(Union{Nothing,AbstractVector}, + newcapacity) + _nestedwritedictreserve!(trace, materialization, newcharge) + sources = try + output = Vector{Union{Nothing,AbstractVector}}(undef, newcapacity) + fill!(output, nothing) + output + catch + _nestedwritedictunreserve!(trace, materialization, newcharge) + rethrow() + end + for source in identities.sources + source === nothing && continue + index = _nestedwritedictidentityslot(sources, source) + sources[index] = source + end + oldcharge = _materializedarraybytes(Union{Nothing,AbstractVector}, + capacity) + identities.sources = sources + _nestedwritedictunreserve!(trace, materialization, oldcharge) + return +end + +function _nestedwritedictidentityinsertset!(trace, + materialization::_NestedWriteDictMaterialization, + identities::_NestedWriteIdentitySet, source::AbstractVector) + index = _nestedwritedictidentityslot(identities.sources, source) + identities.sources[index] === source && return false + _nestedwritedictidentitygrow!(trace, materialization, identities) + index = _nestedwritedictidentityslot(identities.sources, source) + identities.sources[index] = source + identities.count = Base.checked_add(identities.count, 1) + return true +end + +function _nestedwritedictidentityinsert!(trace, + materialization::_NestedWriteDictMaterialization, + source::AbstractVector) + identities = materialization.dependency_sources::_NestedWriteIdentitySet + return _nestedwritedictidentityinsertset!(trace, materialization, + identities, source) +end + +function _nestedwritedictidentitycontains( + materialization::_NestedWriteDictMaterialization, + source::AbstractVector) + identities = materialization.dependency_sources::_NestedWriteIdentitySet + index = _nestedwritedictidentityslot(identities.sources, source) + return identities.sources[index] === source +end + +function _nestedwritedictpreflightinsert!(trace, + materialization::_NestedWriteDictMaterialization, + source::AbstractVector) + identities = materialization.preflight_sources::_NestedWriteIdentitySet + return _nestedwritedictidentityinsertset!(trace, materialization, + identities, source) +end + +function _nestedwritedictpreflightrelease!(trace, + materialization::_NestedWriteDictMaterialization) + identities = materialization.preflight_sources + identities isa _NestedWriteIdentitySet || return + charge = _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _materializedarraybytes(Union{Nothing,AbstractVector}, + length(identities.sources))) + materialization.preflight_sources = nothing + _nestedwritedictunreserve!(trace, materialization, charge) + return +end + +function _nestedwritedictdependencynode!(trace, + materialization::_NestedWriteDictMaterialization, snapshot) + _nestedwritedictreserve!(trace, materialization, + _MATERIALIZED_OBJECT_BYTES) + dependency = try + _NestedWriteDictDependency(snapshot, nothing) + catch + _nestedwritedictunreserve!(trace, materialization, + _MATERIALIZED_OBJECT_BYTES) + rethrow() + end + if materialization.dependency_last === nothing + materialization.dependencies = dependency + else + materialization.dependency_last.next = dependency + end + materialization.dependency_last = dependency + return +end + +function _nestedwritedictdependency!(trace, + materialization::_NestedWriteDictMaterialization, + snapshot::_NestedWriteVectorSnapshot) + _nestedwritedictidentityinsert!(trace, materialization, + snapshot.source) || return + _nestedwritedictdependencynode!(trace, materialization, snapshot) + return +end + +function _nestedwritedictdependency!(trace, + materialization::_NestedWriteDictMaterialization, + snapshot::_NestedWriteStructViewSnapshot) + _nestedwritedictdependencynode!(trace, materialization, snapshot) + return +end + +function _nestedwritedictstructview!(trace, + materialization::_NestedWriteDictMaterialization, value::StructValue, + shape) + names = value.names + children = value.children + count = length(children) + length(names) == count || throw(ArgumentError( + "nested Parquet struct view changed its field count")) + if shape isa _NestedWriteStructShape + names == shape.names && count == length(shape.children) || throw( + ArgumentError( + "nested Parquet struct view does not match its declared topology")) + end + value.index >= 1 || throw(ArgumentError( + "nested Parquet struct view has an invalid row index")) + copycharge = _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _materializedarraybytes(String, count)) + copycharge = _materializedsum(copycharge, + _materializedarraybytes(AbstractVector, count)) + _nestedwritedictreserve!(trace, materialization, copycharge) + snapshot = try + _NestedWriteStructViewSnapshot(value, names, children, copy(names), + copy(children), value.index) + catch + _nestedwritedictunreserve!(trace, materialization, copycharge) + rethrow() + end + _nestedwritedictdependency!(trace, materialization, snapshot) + return +end + +function _nestedwritedictlistviewlocal(value::ListValue) + _nestedviewcount(value.first, value.last, "list view") + return +end + +function _nestedwritedictmapviewlocal( + value::MapValue{K,V,HasValues}) where {K,V,HasValues} + HasValues isa Bool || throw(ArgumentError( + "map view has a non-Boolean value-vector discriminator")) + Missing <: K && throw(ArgumentError( + "map view key type cannot include Missing")) + HasValues === false && V !== Missing && throw(ArgumentError( + "key-only map view must use Missing as its value type")) + if HasValues === true + value.values === nothing && throw(ArgumentError( + "map view lost its value vector")) + else + value.values === nothing || throw(ArgumentError( + "key-only map view gained a value vector")) + end + _nestedviewcount(value.first, value.last, "map view") + return +end + +function _nestedwritedictpreflightlocal(value, shape) + if value isa ListValue + _nestedwritedictlistviewlocal(value) + elseif value isa MapValue + _nestedwritedictmapviewlocal(value) + elseif value isa StructValue + length(value.names) == length(value.children) || throw(ArgumentError( + "nested Parquet struct view changed its field count")) + value.index >= 1 || throw(ArgumentError( + "nested Parquet struct view has an invalid row index")) + if shape isa _NestedWriteStructShape + value.names == shape.names && + length(value.children) == length(shape.children) || throw( + ArgumentError( + "nested Parquet struct view does not match its declared topology")) + end + elseif value isa ListVector + _validatelistlocal(value) + elseif value isa StructVector + _validatestructlocal(value) + elseif value isa MapVector + _validatemaplocal(value) + end + return +end + +function _nestedwritedictpreflightgraph!(trace, + materialization::_NestedWriteDictMaterialization, value, shape, + limits::Limits, depth::Int) + trace isa _NestedWriteTrace || return + work = _nestedwritedictwork(trace, value, shape, depth, false, nothing) + try + while work !== nothing + current = work + work = current.next + item = current.value + itemshape = current.shape + itemdepth = current.depth + current.value = nothing + current.next = nothing + _nestedwritedictworkfree!(trace) + _checklimit(:metadata_depth, itemdepth, + limits.max_metadata_depth) + if item isa Union{ListValue,MapValue} + _nestedwritedictpreflightlocal(item, itemshape) + elseif item isa AbstractVector + _nestedwritedictpreflightinsert!(trace, materialization, + item) || continue + _nestedwritedictpreflightlocal(item, itemshape) + else + _nestedwritedictpreflightlocal(item, itemshape) + end + if item isa ListValue + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + childshape = itemshape isa _NestedWriteListShape ? + itemshape.element : nothing + work = _nestedwritedictwork(trace, item.values, childshape, + childdepth, true, work) + elseif item isa StructValue && !isempty(item.children) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + for index in length(item.children):-1:1 + childshape = itemshape isa _NestedWriteStructShape && + index <= length(itemshape.children) ? + itemshape.children[index] : nothing + work = _nestedwritedictwork(trace, item.children[index], + childshape, childdepth, true, work) + end + elseif item isa MapValue + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + valueshape = itemshape isa _NestedWriteMapShape ? + itemshape.value : nothing + item.values === nothing || (work = _nestedwritedictwork( + trace, item.values, valueshape, childdepth, true, work)) + keyshape = itemshape isa _NestedWriteMapShape ? + itemshape.key : nothing + work = _nestedwritedictwork(trace, item.keys, keyshape, + childdepth, true, work) + elseif item isa Union{LogicalColumn,FixedByteArrayVector} + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + work = _nestedwritedictwork(trace, item.values, itemshape, + childdepth, true, work) + elseif item isa ListVector + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + childshape = itemshape isa _NestedWriteListShape ? + itemshape.element : nothing + work = _nestedwritedictwork(trace, item.values, childshape, + childdepth, true, work) + elseif item isa StructVector && !isempty(item.children) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + for index in length(item.children):-1:1 + childshape = itemshape isa _NestedWriteStructShape && + index <= length(itemshape.children) ? + itemshape.children[index] : nothing + work = _nestedwritedictwork(trace, item.children[index], + childshape, childdepth, true, work) + end + elseif item isa MapVector + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + valueshape = itemshape isa _NestedWriteMapShape ? + itemshape.value : nothing + item.values === nothing || (work = _nestedwritedictwork( + trace, item.values, valueshape, childdepth, true, work)) + keyshape = itemshape isa _NestedWriteMapShape ? + itemshape.key : nothing + work = _nestedwritedictwork(trace, item.keys, keyshape, + childdepth, true, work) + elseif item isa Pair && itemshape === nothing + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + work = _nestedwritedictwork(trace, item.second, nothing, + childdepth, false, work) + work = _nestedwritedictwork(trace, item.first, nothing, + childdepth, false, work) + elseif item isa Union{Tuple,NamedTuple} && + itemshape isa Union{Nothing,_NestedWriteStructShape} && + !isempty(item) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + for index in length(item):-1:1 + childshape = itemshape isa _NestedWriteStructShape && + index <= length(itemshape.children) ? + itemshape.children[index] : nothing + work = _nestedwritedictwork(trace, getfield(item, index), + childshape, childdepth, false, work) + end + elseif itemshape isa _NestedWriteStructShape && + !ismissing(item) && fieldcount(typeof(item)) > 0 + count = min(fieldcount(typeof(item)), + length(itemshape.children)) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + for index in count:-1:1 + work = _nestedwritedictwork(trace, getfield(item, index), + itemshape.children[index], childdepth, false, work) + end + end + end + finally + _nestedwritedictworkcleanup!(trace, work) + end + return +end + +function _nestedwritedictsourceauthority(trace::_NestedWriteTrace, + materialization::_NestedWriteDictMaterialization, + source::AbstractVector) + _nestedwritedictidentitycontains(materialization, source) || throw( + AssertionError( + "nested Parquet dictionary view has no captured source authority")) + topology = trace.topology + topology isa _NestedWriteTopologySnapshot || throw(AssertionError( + "nested Parquet trace has no topology authority")) + snapshot = _nestedwritesnapshotlookupget(topology.lookup, source) + snapshot === nothing && throw(AssertionError( + "nested Parquet dictionary source authority is absent")) + return snapshot +end + +function _nestedwritedictonebased(snapshot::_NestedWriteVectorSnapshot, + label::String) + snapshot.first == 1 && snapshot.last == snapshot.count || throw( + ArgumentError("$label must use one-based contiguous axes")) + return snapshot.count +end + +function _nestedwritedictlistviewauthority(trace::_NestedWriteTrace, + materialization::_NestedWriteDictMaterialization, value::ListValue) + _nestedwritedictlistviewlocal(value) + snapshot = _nestedwritedictsourceauthority(trace, materialization, + value.values) + count = _nestedwritedictonebased(snapshot, "list view backing vector") + (value.first <= count || (count < typemax(Int) && + value.first == count + 1)) || throw(ArgumentError( + "list view insertion point exceeds its backing vector")) + if value.last >= value.first + value.last <= count || throw(ArgumentError( + "list view child span exceeds its backing vector axes")) + end + return +end + +function _nestedwritedictmapviewauthority(trace::_NestedWriteTrace, + materialization::_NestedWriteDictMaterialization, value::MapValue) + _nestedwritedictmapviewlocal(value) + keysnapshot = _nestedwritedictsourceauthority(trace, materialization, + value.keys) + keycount = _nestedwritedictonebased(keysnapshot, "map view key vector") + (value.first <= keycount || (keycount < typemax(Int) && + value.first == keycount + 1)) || throw(ArgumentError( + "map view insertion point exceeds its key vector")) + value.last < value.first || value.last <= keycount || throw( + ArgumentError("map view entry span exceeds its key-vector axes")) + if value.values !== nothing + valuesnapshot = _nestedwritedictsourceauthority(trace, + materialization, value.values) + valuecount = _nestedwritedictonebased(valuesnapshot, + "map view value vector") + valuecount == keycount || throw(ArgumentError( + "map view key and value lengths differ")) + value.last < value.first || value.last <= valuecount || throw( + ArgumentError("map view entry span exceeds its value-vector axes")) + end + return +end + +function _nestedwritedictviewlocal( + snapshot::_NestedWriteStructViewSnapshot) + value = snapshot.source + value.names === snapshot.names && + value.children === snapshot.children && + value.index == snapshot.index || throw(ArgumentError( + "nested Parquet struct view backing identity changed during dictionary iteration")) + length(value.names) == length(snapshot.names_copy) && + length(value.children) == length(snapshot.children_copy) || throw( + ArgumentError( + "nested Parquet struct view field count changed during dictionary iteration")) + _nestedwritearrayequal(value.names, snapshot.names_copy) || throw( + ArgumentError( + "nested Parquet struct view names changed during dictionary iteration")) + _nestedwritevectoridentities(value.children, + snapshot.children_copy) || throw(ArgumentError( + "nested Parquet struct view child identity or order changed during dictionary iteration")) + return +end + +function _nestedwritedictdependencylocal( + snapshot::_NestedWriteVectorSnapshot) + _nestedwritevalidatenodelocal(snapshot) + return +end + +function _nestedwritedictdependencylocal( + snapshot::_NestedWriteStructViewSnapshot) + _nestedwritedictviewlocal(snapshot) + return +end + +function _nestedwritedictdependencyfull( + snapshot::_NestedWriteVectorSnapshot) + _nestedwritevalidatenode(snapshot) + return +end + +function _nestedwritedictdependencyfull( + snapshot::_NestedWriteStructViewSnapshot) + _nestedwritedictviewlocal(snapshot) + _validatestructvalue(snapshot.source) + _nestedwritedictviewlocal(snapshot) + return +end + +function _nestedwritedictwork(trace, value, shape, depth::Int, source::Bool, + next) + trace isa _NestedWriteTrace && + _nestedwritetracereserve!(trace, _MATERIALIZED_OBJECT_BYTES) + try + return _NestedWriteDictWork(value, shape, depth, source, next) + catch + trace isa _NestedWriteTrace && + _nestedwritetraceunreserve!(trace, _MATERIALIZED_OBJECT_BYTES) + rethrow() + end +end + +function _nestedwritedictworkfree!(trace) + trace isa _NestedWriteTrace && + _nestedwritetraceunreserve!(trace, _MATERIALIZED_OBJECT_BYTES) + return +end + +function _nestedwritedictworkcleanup!(trace, work) + current = work + while current !== nothing + following = current.next + current.value = nothing + current.next = nothing + _nestedwritedictworkfree!(trace) + current = following + end + return +end + +function _nestedwritedictsnapshot!(trace::_NestedWriteTrace, + source::AbstractVector, limits::Limits) + topology = trace.topology + topology isa _NestedWriteTopologySnapshot || throw(AssertionError( + "nested Parquet trace has no topology authority")) + snapshot = if trace.capturing + _nestedwriteextendtopology!(topology, source, limits, trace.budget) + else + stored = _nestedwritesnapshotlookupget(topology.lookup, source) + stored === nothing && throw(ArgumentError( + "nested Parquet dictionary changed a source identity between writer passes")) + stored + end + _nestedwritevalidatenodelocal(snapshot) + return snapshot +end + +function _nestedwritedictsnapshotlocal!(trace::_NestedWriteTrace, + source::AbstractVector) + topology = trace.topology + topology isa _NestedWriteTopologySnapshot || throw(AssertionError( + "nested Parquet trace has no topology authority")) + snapshot = if trace.capturing + _nestedwriteextendtopologylocal!(topology, source, trace.budget) + else + stored = _nestedwritesnapshotlookupget(topology.lookup, source) + stored === nothing && throw(ArgumentError( + "nested Parquet dictionary changed a source identity between writer passes")) + stored + end + _nestedwritevalidatenodelocal(snapshot) + return snapshot +end + +function _nestedwritedictpushsource!(trace, + materialization::_NestedWriteDictMaterialization, work, value, + shape, depth::Int, limits::Limits) + _nestedwritedictidentitycontains(materialization, value) && return work + snapshot = _nestedwritedictsnapshot!(trace, value, limits) + _nestedwritedictdependency!(trace, materialization, snapshot) + if snapshot.kind in (_NESTED_WRITE_VECTOR_LOGICAL, + _NESTED_WRITE_VECTOR_FIXED) + childdepth = _nestedwritekeynextdepth(depth, limits) + work = _nestedwritedictwork(trace, snapshot.primary, shape, + childdepth, true, work) + elseif snapshot.kind == _NESTED_WRITE_VECTOR_LIST + childdepth = _nestedwritekeynextdepth(depth, limits) + childshape = shape isa _NestedWriteListShape ? shape.element : nothing + work = _nestedwritedictwork(trace, snapshot.tertiary, childshape, + childdepth, true, work) + elseif snapshot.kind == _NESTED_WRITE_VECTOR_STRUCT && + !isempty(snapshot.copy3) + childdepth = _nestedwritekeynextdepth(depth, limits) + children = snapshot.copy3 + for index in length(children):-1:1 + childshape = shape isa _NestedWriteStructShape && + index <= length(shape.children) ? shape.children[index] : + nothing + work = _nestedwritedictwork(trace, children[index], childshape, + childdepth, true, work) + end + elseif snapshot.kind == _NESTED_WRITE_VECTOR_MAP + childdepth = _nestedwritekeynextdepth(depth, limits) + valueshape = shape isa _NestedWriteMapShape ? shape.value : nothing + snapshot.quaternary === nothing || (work = _nestedwritedictwork( + trace, snapshot.quaternary, valueshape, childdepth, true, work)) + keyshape = shape isa _NestedWriteMapShape ? shape.key : nothing + work = _nestedwritedictwork(trace, snapshot.tertiary, keyshape, + childdepth, true, work) + end + return work +end + +function _nestedwritedictpushsourcelocal!(trace, + materialization::_NestedWriteDictMaterialization, work, value, + shape, depth::Int, limits::Limits) + _nestedwritedictidentitycontains(materialization, value) && return work + snapshot = _nestedwritedictsnapshotlocal!(trace, value) + snapshot === nothing && return work + _nestedwritedictdependency!(trace, materialization, snapshot) + if snapshot.kind in (_NESTED_WRITE_VECTOR_LOGICAL, + _NESTED_WRITE_VECTOR_FIXED) + childdepth = _nestedwritekeynextdepth(depth, limits) + work = _nestedwritedictwork(trace, snapshot.primary, shape, + childdepth, true, work) + elseif snapshot.kind == _NESTED_WRITE_VECTOR_LIST + childdepth = _nestedwritekeynextdepth(depth, limits) + childshape = shape isa _NestedWriteListShape ? shape.element : nothing + work = _nestedwritedictwork(trace, snapshot.tertiary, childshape, + childdepth, true, work) + elseif snapshot.kind == _NESTED_WRITE_VECTOR_STRUCT && + !isempty(snapshot.copy3) + childdepth = _nestedwritekeynextdepth(depth, limits) + children = snapshot.copy3 + for index in length(children):-1:1 + childshape = shape isa _NestedWriteStructShape && + index <= length(shape.children) ? shape.children[index] : + nothing + work = _nestedwritedictwork(trace, children[index], childshape, + childdepth, true, work) + end + elseif snapshot.kind == _NESTED_WRITE_VECTOR_MAP + childdepth = _nestedwritekeynextdepth(depth, limits) + valueshape = shape isa _NestedWriteMapShape ? shape.value : nothing + snapshot.quaternary === nothing || (work = _nestedwritedictwork( + trace, snapshot.quaternary, valueshape, childdepth, true, work)) + keyshape = shape isa _NestedWriteMapShape ? shape.key : nothing + work = _nestedwritedictwork(trace, snapshot.tertiary, keyshape, + childdepth, true, work) + end + return work +end + +function _nestedwritedictcapturelocal!(trace, + materialization::_NestedWriteDictMaterialization, value, shape, + limits::Limits, depth::Int) + trace isa _NestedWriteTrace || return + work = _nestedwritedictwork(trace, value, shape, depth, false, nothing) + try + while work !== nothing + current = work + work = current.next + item = current.value + itemshape = current.shape + itemdepth = current.depth + source = current.source + current.value = nothing + current.next = nothing + _nestedwritedictworkfree!(trace) + _checklimit(:metadata_depth, itemdepth, + limits.max_metadata_depth) + if item isa ListValue + _nestedwritedictlistviewlocal(item) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + childshape = itemshape isa _NestedWriteListShape ? + itemshape.element : nothing + work = _nestedwritedictwork(trace, item.values, childshape, + childdepth, true, work) + elseif item isa StructValue + _nestedwritedictstructview!(trace, materialization, item, + itemshape) + if !isempty(item.children) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + for index in length(item.children):-1:1 + childshape = itemshape isa _NestedWriteStructShape && + index <= length(itemshape.children) ? + itemshape.children[index] : nothing + work = _nestedwritedictwork(trace, + item.children[index], childshape, childdepth, + true, work) + end + end + elseif item isa MapValue + _nestedwritedictmapviewlocal(item) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + valueshape = itemshape isa _NestedWriteMapShape ? + itemshape.value : nothing + item.values === nothing || (work = _nestedwritedictwork( + trace, item.values, valueshape, childdepth, true, work)) + keyshape = itemshape isa _NestedWriteMapShape ? + itemshape.key : nothing + work = _nestedwritedictwork(trace, item.keys, keyshape, + childdepth, true, work) + elseif item isa AbstractVector && + (source || _nestedwriteispackagevector(item)) + work = _nestedwritedictpushsourcelocal!(trace, + materialization, work, item, itemshape, itemdepth, limits) + elseif item isa Pair && itemshape === nothing + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + work = _nestedwritedictwork(trace, item.second, nothing, + childdepth, false, work) + work = _nestedwritedictwork(trace, item.first, nothing, + childdepth, false, work) + elseif item isa Union{Tuple,NamedTuple} && + itemshape isa Union{Nothing,_NestedWriteStructShape} && + !isempty(item) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + for index in length(item):-1:1 + childshape = itemshape isa _NestedWriteStructShape && + index <= length(itemshape.children) ? + itemshape.children[index] : nothing + work = _nestedwritedictwork(trace, getfield(item, index), + childshape, childdepth, false, work) + end + elseif itemshape isa _NestedWriteStructShape && + !ismissing(item) && fieldcount(typeof(item)) > 0 + count = min(fieldcount(typeof(item)), + length(itemshape.children)) + childdepth = _nestedwritekeynextdepth(itemdepth, limits) + for index in count:-1:1 + work = _nestedwritedictwork(trace, getfield(item, index), + itemshape.children[index], childdepth, false, work) + end + end + end + finally + _nestedwritedictworkcleanup!(trace, work) + end + return +end + +function _nestedwritedictdiscover!(trace, + materialization::_NestedWriteDictMaterialization, value, shape, + limits::Limits, depth::Int) + trace isa _NestedWriteTrace || return + work = _nestedwritedictwork(trace, value, shape, depth, false, nothing) + try + while work !== nothing + current = work + work = current.next + item = current.value + itemshape = current.shape + depth = current.depth + source = current.source + current.value = nothing + current.next = nothing + _nestedwritedictworkfree!(trace) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + if item isa ListValue + _nestedwritedictlistviewauthority(trace, materialization, + item) + elseif item isa StructValue + _nestedwritedictpreflightlocal(item, itemshape) + elseif item isa MapValue + _nestedwritedictmapviewauthority(trace, materialization, + item) + elseif item isa AbstractVector && + (source || _nestedwriteispackagevector(item)) + work = _nestedwritedictpushsource!(trace, materialization, + work, item, itemshape, depth, limits) + elseif item isa Pair && itemshape === nothing + childdepth = _nestedwritekeynextdepth(depth, limits) + work = _nestedwritedictwork(trace, item.second, nothing, + childdepth, false, work) + work = _nestedwritedictwork(trace, item.first, nothing, + childdepth, false, work) + elseif item isa Union{Tuple,NamedTuple} && + itemshape isa Union{Nothing,_NestedWriteStructShape} + childdepth = _nestedwritekeynextdepth(depth, limits) + for index in length(item):-1:1 + childshape = itemshape isa _NestedWriteStructShape && + index <= length(itemshape.children) ? + itemshape.children[index] : nothing + work = _nestedwritedictwork(trace, getfield(item, index), + childshape, childdepth, false, work) + end + elseif itemshape isa _NestedWriteStructShape && + !ismissing(item) + count = min(fieldcount(typeof(item)), + length(itemshape.children)) + childdepth = _nestedwritekeynextdepth(depth, limits) + for index in count:-1:1 + work = _nestedwritedictwork(trace, getfield(item, index), + itemshape.children[index], childdepth, false, work) + end + end + end + finally + _nestedwritedictworkcleanup!(trace, work) + end + return +end + +function _nestedwritedictvalidatelocal!( + materialization::_NestedWriteDictMaterialization) + dependency = materialization.dependencies + while dependency !== nothing + _nestedwritedictdependencylocal(dependency.snapshot) + dependency = dependency.next + end + return +end + +function _nestedwritedictvalidate!( + materialization::_NestedWriteDictMaterialization) + _nestedwritedictvalidatelocal!(materialization) + dependency = materialization.dependencies + while dependency !== nothing + _nestedwritedictdependencyfull(dependency.snapshot) + dependency = dependency.next + end + _nestedwritedictvalidatelocal!(materialization) + return +end + +function _nestedwritedictmaterialize(trace, value::AbstractDict, keyshape, + valueshape, limits::Limits, depth::Int=1) + materialization = _nestedwritedictmaterialization(trace) + try + result = iterate(value) + while result !== nothing + result isa Tuple && length(result) == 2 || throw(ArgumentError( + "Parquet MAP dictionary returned an invalid iteration result")) + pair = result[1] + pair isa Pair || throw(ArgumentError( + "Parquet MAP dictionary must iterate Pair values")) + ismissing(pair.first) && throw(ArgumentError( + "Parquet MAP dictionary contains a missing key")) + _nestedwritekeydeclared(keyshape, pair.first) + _nestedwritekeydeclared(valueshape, pair.second) + state = result[2] + count = Base.checked_add(materialization.count, 1) + _checklimit(:container_elements, count, + limits.max_container_elements) + _nestedwritedictpreflightgraph!(trace, materialization, + pair.first, keyshape, limits, depth) + _nestedwritedictpreflightgraph!(trace, materialization, + pair.second, valueshape, limits, depth) + _nestedwritedictentry!(trace, materialization, pair) + materialization.count = count + _nestedwritedictcapturelocal!(trace, materialization, pair.first, + keyshape, limits, depth) + _nestedwritedictcapturelocal!(trace, materialization, pair.second, + valueshape, limits, depth) + result = iterate(value, state) + end + _nestedwritedictpreflightrelease!(trace, materialization) + _nestedwritedictvalidatelocal!(materialization) + entry = materialization.first + while entry !== nothing + _nestedwritedictdiscover!(trace, materialization, entry.key, + keyshape, limits, depth) + _nestedwritedictdiscover!(trace, materialization, entry.value, + valueshape, limits, depth) + entry = entry.next + end + _nestedwritedictvalidate!(materialization) + return materialization + catch + _nestedwritedictrelease!(trace, materialization) + rethrow() + end +end + +function _nestedwriteinputrawname(column::Pair) + return first(column) +end + +function _nestedwriteinputrawname(column) + hasproperty(column, :name) || throw(ArgumentError( + "nested writer input columns need name and values fields")) + return getproperty(column, :name) +end + +function _nestedwritevalidateinput(snapshot::_NestedWriteTopologySnapshot, + budget::_LiveByteBudget) + snapshot.input_columns === nothing && return + start = _budgetused(budget) + try + columns = snapshot.input_columns + length(columns) == snapshot.input_count || throw(ArgumentError( + "nested Parquet top-level column count changed between writer phases")) + firstindex(columns) == snapshot.input_first && + lastindex(columns) == snapshot.input_last || throw(ArgumentError( + "nested Parquet top-level column axes changed between writer phases")) + position = 0 + for column in columns + position += 1 + position <= length(snapshot.names) || throw(ArgumentError( + "nested Parquet top-level columns changed between writer phases")) + raw = _nestedwriteinputrawname(column) + bytes = _writecolumnnamebytes(raw) + temporary = _materializedsum(_MATERIALIZED_OBJECT_BYTES, bytes) + _reserve!(budget, temporary) + matches = _writecolumnnameequal(raw, snapshot.names[position]) + _release!(budget, temporary) + matches || throw(ArgumentError( + "nested Parquet top-level column name or order changed between writer phases")) + _nestedwriteinputvalues(column) === snapshot.values[position] || + throw(ArgumentError( + "nested Parquet top-level column identity changed between writer phases")) + end + position == length(snapshot.names) || throw(ArgumentError( + "nested Parquet top-level columns changed between writer phases")) + return + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _nestedwritebarrier!(snapshot::_NestedWriteTopologySnapshot, + sourcevalidator, budget::_LiveByteBudget) + _validatewriteinput(sourcevalidator, budget) + _nestedwritevalidateinput(snapshot, budget) + for node in snapshot.nodes + _nestedwritevalidatenode(node) + end + return +end + +function _nestedwritetopologyrelease!(snapshot::_NestedWriteTopologySnapshot, + budget::_LiveByteBudget) + iszero(snapshot.charge) || _release!(budget, snapshot.charge) + return +end + +struct _NestedWriteCountContext + counts::Vector{_NestedWriteLeafCount} + limits::Limits + semantic::Union{Nothing,_NestedSchemaPlan} + entry_offsets::Union{Nothing,Vector{Vector{Int64}}} + dense_offsets::Union{Nothing,Vector{Vector{Int64}}} + payload_offsets::Union{Nothing,Vector{Vector{Int64}}} + trace::Union{Nothing,_NestedWriteTrace} +end + +function _NestedWriteCountContext(counts::Vector{_NestedWriteLeafCount}, + limits::Limits) + return _NestedWriteCountContext(counts, limits, nothing, nothing, nothing, + nothing, nothing) +end + +struct _NestedWriteEmitContext + builders::Vector{_NestedWriteLeafBuilder} + counts::Vector{_NestedWriteLeafCount} + limits::Limits + boundaries::Union{Nothing,_NestedWriteCountContext} + trace::Union{Nothing,_NestedWriteTrace} +end + +function _NestedWriteEmitContext(builders::Vector{_NestedWriteLeafBuilder}, + counts::Vector{_NestedWriteLeafCount}, limits::Limits) + return _NestedWriteEmitContext(builders, counts, limits, nothing, nothing) +end + +function _nestedwriteinputname(column::Pair) + return String(first(column)) +end + +function _nestedwriteinputname(column) + hasproperty(column, :name) || throw(ArgumentError( + "nested writer input columns need name and values fields")) + return String(getproperty(column, :name)) +end + +function _nestedwriteinputvalues(column::Pair) + return last(column) +end + +function _nestedwriteinputvalues(column) + hasproperty(column, :values) || throw(ArgumentError( + "nested writer input columns need name and values fields")) + return getproperty(column, :values) +end + +function _nestedwritesplittype(declared::Type; key::Bool=false) + declared === Any && throw(ArgumentError( + "nested Parquet writer types cannot be Any")) + declared === Union{} && throw(ArgumentError( + "nested Parquet writer types cannot be Union{}")) + members = Base.uniontypes(declared) + if length(members) == 1 + value_type = only(members) + if value_type === Missing + key && throw(ArgumentError("Parquet MAP key types cannot include Missing")) + return Missing, true + end + value_type === Nothing && throw(ArgumentError( + "nothing is not a Parquet null value; use missing")) + return value_type, false + end + length(members) == 2 && Missing in members || throw(ArgumentError( + "nested Parquet writer types must be T or Union{Missing,T}, got $declared")) + key && throw(ArgumentError("Parquet MAP key types cannot include Missing")) + value_type = members[1] === Missing ? members[2] : members[1] + value_type === Nothing && throw(ArgumentError( + "nothing is not a Parquet null value; use missing")) + return value_type, true +end + +function _nestedwriteisfixedtuple(::Type{T}) where {T} + isconcretetype(T) || return false + T <: Tuple || return false + count = fieldcount(T) + count > 0 || return false + for field in fieldtypes(T) + field === UInt8 || return false + end + return true +end + +function _nestedwriteisscalar(value_type::Type) + value_type === Missing && return true + value_type in (Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, + UInt64, Float16, Float32, Float64, Dates.Date, Dates.Time, + Dates.DateTime, Decimal, UUIDs.UUID, JSONValue, BSONValue, Interval) && + return true + value_type <: Timestamp && return true + value_type <: AbstractString && return true + value_type <: AbstractVector{UInt8} && return true + return _nestedwriteisfixedtuple(value_type) +end + +function _nestedwriteleafshape(name::String, value_type::Type, optional::Bool, + budget::_LiveByteBudget; explicit=nothing, fixed_width=nothing, + source_snapshot=nothing) + _reserveobjects!(budget, 2) + aggregate = _NestedWriteLeafAggregate(false, nothing, nothing, Int32(1)) + return _NestedWriteLeafShape(name, value_type, optional, explicit, + fixed_width, aggregate, source_snapshot) +end + +function _nestedwriteshapesnapshot(source, + topology::Union{Nothing,_NestedWriteTopologySnapshot}) + source isa AbstractVector && topology !== nothing || return nothing + return _nestedwritesnapshotfor(topology, source) +end + +function _nestedwritecheckconcrete(value_type::Type, label::AbstractString) + isconcretetype(value_type) && return + throw(ArgumentError("$label must use a concrete declared type, got $value_type")) +end + +function _nestedwriteshapestart(name::String, declared::Type, source, + limits::Limits, budget::_LiveByteBudget, key::Bool, depth::Int, + topology::Union{Nothing,_NestedWriteTopologySnapshot}) + Base.@nospecialize declared source + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + value_type, optional = _nestedwritesplittype(declared; key=key) + if source isa LogicalColumn + key && optional && throw(ArgumentError( + "Parquet MAP key types cannot include Missing")) + return _nestedwriteleafshape(name, value_type, optional, budget; + explicit=source.spec, + source_snapshot=_nestedwriteshapesnapshot(source, topology)) + elseif source isa FixedByteArrayVector + source.width > 0 || throw(ArgumentError( + "fixed byte-array width must be positive")) + return _nestedwriteleafshape(name, value_type, optional, budget; + fixed_width=source.width, + source_snapshot=_nestedwriteshapesnapshot(source, topology)) + elseif source isa StructVector + isempty(source.names) && throw(ArgumentError( + "zero-field structs cannot be written to Parquet")) + packageoptional = source.ranks !== nothing + packageoptional == optional || throw(ArgumentError( + "StructVector validity does not match its declared element type")) + _checklimit(:container_elements, length(source.children), + limits.max_container_elements) + _reservearray!(budget, String, length(source.names)) + _reservearray!(budget, _NestedWriteShape, length(source.children)) + _reservearray!(budget, AbstractVector, length(source.children)) + names = copy(source.names) + sourcechildren = copy(source.children) + children = _NestedWriteShape[] + sizehint!(children, length(source.children)) + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_STRUCT, name, + StructValue, optional, source, depth, names, sourcechildren, + nothing, nothing, nothing, children, nothing, nothing, 0, true, + true) + elseif source isa ListVector + packageoptional = source.validity !== nothing + packageoptional == optional || throw(ArgumentError( + "ListVector validity does not match its declared element type")) + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_LIST, name, + value_type, optional, source, depth, nothing, nothing, + eltype(source.values), source.values, nothing, nothing, nothing, + nothing, 0, true, true) + elseif source isa MapVector + packageoptional = source.validity !== nothing + packageoptional == optional || throw(ArgumentError( + "MapVector validity does not match its declared element type")) + values = source.values + childtypes = (eltype(source.keys), + values === nothing ? Missing : eltype(values)) + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_MAP, name, + value_type, optional, source, depth, nothing, nothing, childtypes, + source.keys, values, nothing, nothing, nothing, 0, true, + values !== nothing) + elseif value_type <: ListValue + _nestedwritecheckconcrete(value_type, "Parquet list view type") + elementtype = eltype(value_type) + elementtype === Any && throw(ArgumentError( + "Parquet list element types cannot be Any")) + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_LIST, name, + value_type, optional, source, depth, nothing, nothing, elementtype, + nothing, nothing, nothing, nothing, nothing, 0, true, true) + end + if _nestedwriteisscalar(value_type) + value_type === Missing || _nestedwritecheckconcrete(value_type, + "Parquet scalar type") + snapshot = _nestedwriteshapesnapshot(source, topology) + return _nestedwriteleafshape(name, value_type, optional, budget; + source_snapshot=snapshot) + elseif value_type <: StructValue + throw(ArgumentError( + "StructValue needs its owning StructVector to declare field names and types")) + elseif value_type <: NamedTuple + _nestedwritecheckconcrete(value_type, "Parquet struct type") + count = fieldcount(value_type) + iszero(count) && throw(ArgumentError( + "zero-field structs cannot be written to Parquet")) + _checklimit(:container_elements, count, + limits.max_container_elements) + _reservearray!(budget, String, count) + names = String[String(field) for field in fieldnames(value_type)] + types = fieldtypes(value_type) + _reservearray!(budget, _NestedWriteShape, length(names)) + children = _NestedWriteShape[] + sizehint!(children, length(names)) + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_STRUCT, name, + value_type, optional, source, depth, names, nothing, types, nothing, + nothing, children, nothing, nothing, 0, false, true) + elseif value_type <: MapValue + _nestedwritecheckconcrete(value_type, "Parquet map view type") + pairtype = eltype(value_type) + pairtype <: Pair || throw(ArgumentError( + "MapValue has no concrete key and value types")) + mapkeytype = pairtype.parameters[1] + mapvaluetype = pairtype.parameters[2] + hasvalues = value_type.parameters[3] + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_MAP, name, + value_type, optional, source, depth, nothing, nothing, + (mapkeytype, mapvaluetype), nothing, nothing, nothing, nothing, + nothing, 0, true, hasvalues === true) + elseif value_type <: AbstractDict + _nestedwritecheckconcrete(value_type, "Parquet map type") + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_MAP, name, + value_type, optional, source, depth, nothing, nothing, + (Base.keytype(value_type), Base.valtype(value_type)), nothing, + nothing, nothing, nothing, nothing, 0, false, true) + elseif value_type <: AbstractVector + _nestedwritecheckconcrete(value_type, "Parquet list type") + elementtype = eltype(value_type) + elementtype === Any && throw(ArgumentError( + "Parquet list element types cannot be Any")) + return _NestedWriteShapeFrame(_NESTED_WRITE_SHAPE_LIST, name, + value_type, optional, source, depth, nothing, nothing, elementtype, + nothing, nothing, nothing, nothing, nothing, 0, false, true) + end + throw(ArgumentError("unsupported nested Parquet writer type $value_type")) +end + +function _nestedwriteshapeframeaccept(frame::_NestedWriteShapeFrame, + child::_NestedWriteShape) + children = frame.children + first = frame.first + second = frame.second + if frame.kind == _NESTED_WRITE_SHAPE_STRUCT + push!(something(children), child) + elseif frame.position == 0 + first = child + else + second = child + end + return _NestedWriteShapeFrame(frame.kind, frame.name, frame.value_type, + frame.optional, frame.source, frame.depth, frame.names, + frame.source_children, frame.child_types, frame.first_source, + frame.second_source, children, first, second, frame.position + 1, + frame.package_owned, frame.source_has_values) +end + +function _nestedwriteshapeframeexpected(frame::_NestedWriteShapeFrame) + frame.kind == _NESTED_WRITE_SHAPE_STRUCT && + return length(something(frame.names)) + frame.kind == _NESTED_WRITE_SHAPE_LIST && return 1 + return 2 +end + +function _nestedwriteshapeframenext(frame::_NestedWriteShapeFrame, + limits::Limits) + position = frame.position + 1 + if frame.kind == _NESTED_WRITE_SHAPE_STRUCT + names = something(frame.names) + sourcechildren = frame.source_children + if sourcechildren === nothing + return names[position], frame.child_types[position], nothing, false, + _nestedwritedepthadd(frame.depth, 1, limits) + end + child = sourcechildren[position] + return names[position], eltype(child), child, false, + _nestedwritedepthadd(frame.depth, 1, limits) + elseif frame.kind == _NESTED_WRITE_SHAPE_LIST + return "element", frame.child_types::Type, frame.first_source, false, + _nestedwritedepthadd(frame.depth, 2, limits) + end + types = frame.child_types + if position == 1 + return "key", types[1], frame.first_source, true, + _nestedwritedepthadd(frame.depth, 2, limits) + end + return "value", types[2], frame.second_source, false, + _nestedwritedepthadd(frame.depth, 2, limits) +end + +function _nestedwriteshapeframefinish(frame::_NestedWriteShapeFrame, + budget::_LiveByteBudget, + topology::Union{Nothing,_NestedWriteTopologySnapshot}) + _reserveobjects!(budget) + snapshot = _nestedwriteshapesnapshot(frame.source, topology) + if frame.kind == _NESTED_WRITE_SHAPE_STRUCT + return _NestedWriteStructShape(frame.name, frame.value_type, + frame.optional, something(frame.names), something(frame.children), + frame.source_children, snapshot, frame.package_owned) + elseif frame.kind == _NESTED_WRITE_SHAPE_LIST + return _NestedWriteListShape(frame.name, frame.value_type, + frame.optional, something(frame.first), snapshot) + end + return _NestedWriteMapShape(frame.name, frame.value_type, frame.optional, + something(frame.first), something(frame.second), snapshot, + frame.package_owned, frame.source_has_values) +end + +function _nestedwriteshape(name::String, declared::Type, source, + limits::Limits, budget::_LiveByteBudget; key::Bool=false, + depth::Int=2, + topology::Union{Nothing,_NestedWriteTopologySnapshot}=nothing) + Base.@nospecialize declared source + stack = _nestedwritepassstackstart(_NestedWriteShapeFrame, budget) + pending::Union{Nothing,_NestedWriteShape} = nothing + try + started = _nestedwriteshapestart(name, declared, source, limits, budget, + key, depth, topology) + if started isa _NestedWriteShape + return started + end + _nestedwritestackpush!(stack, started::_NestedWriteShapeFrame, budget) + while true + if pending !== nothing + stack[end] = _nestedwriteshapeframeaccept(stack[end], pending) + pending = nothing + end + frame = stack[end] + if frame.position == _nestedwriteshapeframeexpected(frame) + pending = _nestedwriteshapeframefinish(frame, budget, topology) + _nestedwritestackpop!(stack, budget) + isempty(stack) && return pending + continue + end + childname, childtype, childsource, childkey, childdepth = + _nestedwriteshapeframenext(frame, limits) + started = _nestedwriteshapestart(childname, childtype, childsource, + limits, budget, childkey, childdepth, topology) + if started isa _NestedWriteShape + pending = started + else + _nestedwritestackpush!(stack, + started::_NestedWriteShapeFrame, budget) + end + end + finally + _nestedwritepassstackrelease!(stack, budget) + end +end + +function _nestedwritepresent(shape::_NestedWriteShape, value) + if ismissing(value) + getfield(shape, :optional) || throw(ArgumentError( + "required Parquet field $(repr(getfield(shape, :name))) is missing")) + return false + end + value === nothing && throw(ArgumentError( + "nothing is not a Parquet null value; use missing")) + return true +end + +function _nestedwritecheckvalue(shape::_NestedWriteLeafShape, value) + shape.value_type === Missing && throw(ArgumentError( + "UNKNOWN Parquet field $(repr(shape.name)) can contain only missing")) + value isa shape.value_type || throw(ArgumentError( + "Parquet field $(repr(shape.name)) contains $(typeof(value)); " * + "expected $(shape.value_type)")) + return +end + +function _nestedwritecheckcontainer(value::AbstractVector, limits::Limits, + label::AbstractString) + _checklimit(:container_elements, _nestedvectorcount(value, label), + limits.max_container_elements) + return +end + +function _nestedwriteviewaccesscheck(value::AbstractVector, count::Int, + first::Int, last::Int) + current = _nestedvectorcount(value, "nested Parquet container") + currentfirst, currentlast = _nestedvectoraxes(value, + "nested Parquet container") + current == count && currentfirst == first && currentlast == last || throw(ArgumentError( + "nested Parquet container changed length or axes during access")) + value isa ListValue && _validatelistvalue(value) + value isa MapValue && _validatemapvalue(value) + return +end + +function _nestedwritelistaccessitem(value::AbstractVector, index::Int) + return value[index] +end + +function _nestedwritelistaccessitem(value::ListValue, index::Int) + _validatelistvalue(value) + checkbounds(Bool, value, index) || throw(ArgumentError( + "nested Parquet list index changed during access")) + physical = value.first + index - 1 + child = value.values + checkbounds(Bool, child, physical) || throw(ArgumentError( + "nested Parquet list child no longer contains its physical index")) + return child[physical] +end + +function _nestedwritelistrowsnapshot!(trace, value, + shape::_NestedWriteListShape, limits::Limits) + if value isa ListValue + return _nestedwriteoccurrencesnapshot!(trace, value.values, + shape.element, limits) + elseif _nestedwriteispackagevector(value) + return _nestedwriteoccurrencesnapshot!(trace, value, shape, limits) + end + return nothing +end + +function _nestedwritelistrowaccess(value, index::Int, snapshot) + if value isa ListValue + physical = value.first + index - 1 + return _nestedwriterowaccess(value.values, physical, snapshot) + elseif snapshot isa _NestedWriteVectorSnapshot + return _nestedwriterowaccess(value, index, snapshot) + end + return _nestedwritelistaccessitem(value, index), nothing +end + +function _nestedwritestructaccesschild(value::StructValue, + names::Vector{String}, children::Vector{AbstractVector}, count::Int, + index::Int, expectedname::String, + expectedchild::Union{Nothing,AbstractVector}) + value.index >= 1 || throw(ArgumentError( + "nested Parquet struct row index changed during access")) + value.names === names && length(names) == count || throw(ArgumentError( + "nested Parquet struct names changed during access")) + value.children === children && length(children) == count || throw( + ArgumentError( + "nested Parquet struct child identity or order changed during access")) + 1 <= index <= count || throw(ArgumentError( + "nested Parquet struct child index changed during access")) + names[index] == expectedname || throw(ArgumentError( + "nested Parquet struct field name or order changed during access")) + child = children[index] + expectedchild === nothing || child === expectedchild || throw(ArgumentError( + "nested Parquet struct child identity or order changed during access")) + childcount = _nestedvectorcount(child, "nested Parquet struct child") + childfirst, childlast = _nestedvectoraxes(child, + "nested Parquet struct child") + childfirst == 1 && childlast == childcount || throw(ArgumentError( + "nested Parquet struct child axes changed during access")) + value.index <= childcount || throw(ArgumentError( + "nested Parquet struct child no longer contains its row index")) + return child +end + +function _nestedwritescanleaf!(shape::_NestedWriteLeafShape, value, trace, + row_witness) + _nestedwriterowcheck(row_witness) + present = _nestedwritepresent(shape, value) + _nestedwritetraceleaf!(trace, value, present) + present || return + _nestedwritecheckvalue(shape, value) + shape.explicit === nothing || return + aggregate = shape.aggregate + if shape.value_type == Decimal + decimal = value::Decimal + if aggregate.scale === nothing + aggregate.scale = decimal.scale + elseif aggregate.scale != decimal.scale + throw(ArgumentError("all values in DECIMAL field $(repr(shape.name)) " * + "must use the same scale")) + end + digits = _decimaldigits(decimal.unscaled) + digits <= typemax(Int32) || throw(ArgumentError( + "DECIMAL precision exceeds Int32")) + aggregate.precision = max(aggregate.precision, Int32(digits), + decimal.scale) + aggregate.seen = true + elseif shape.value_type <: Timestamp + timestamp = value::Timestamp + if aggregate.adjusted === nothing + aggregate.adjusted = timestamp.is_adjusted_to_utc + elseif aggregate.adjusted != timestamp.is_adjusted_to_utc + throw(ArgumentError("all values in TIMESTAMP field " * + "$(repr(shape.name)) must use the same UTC adjustment")) + end + aggregate.seen = true + end + return +end + +function _nestedwritescanenter(shape::_NestedWriteShape, value, row_witness) + return _NestedWriteScanAction(_NESTED_WRITE_SCAN_ENTER, shape, value, + row_witness, nothing, nothing, nothing, nothing, nothing, nothing, 0, + 0, 0, 0) +end + +function _nestedwritescanpost(shape::_NestedWriteShape, first, second) + return _NestedWriteScanAction(_NESTED_WRITE_SCAN_POSTCHECK, shape, nothing, + first, second, nothing, nothing, nothing, nothing, nothing, 0, 0, 0, + 0) +end + +function _nestedwritescanstruct!( + stack::_NestedWritePassStack{_NestedWriteScanAction}, + shape::_NestedWriteStructShape, value, trace, row_witness, + budget::_LiveByteBudget) + _nestedwriterowcheck(row_witness) + present = _nestedwritepresent(shape, value) + _nestedwritetracestruct!(trace, value, present, length(shape.children)) + present || return + if shape.package_owned + value isa StructValue || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) requires StructValue rows")) + value.names == shape.names || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) changed its field names")) + length(value) == length(shape.children) || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) changed its field count")) + else + value isa shape.value_type || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) contains $(typeof(value)); " * + "expected $(shape.value_type)")) + end + _nestedwritestackpush!(stack, _NestedWriteScanAction( + _NESTED_WRITE_SCAN_STRUCT, shape, value, row_witness, nothing, nothing, + nothing, nothing, nothing, nothing, 1, length(shape.children), 0, 0), + budget) + return +end + +function _nestedwritescanlist!( + stack::_NestedWritePassStack{_NestedWriteScanAction}, + shape::_NestedWriteListShape, value, limits::Limits, trace, + row_witness, budget::_LiveByteBudget) + _nestedwriterowcheck(row_witness) + value isa ListValue && _validatelistvalue(value) + present = _nestedwritepresent(shape, value) + _nestedwritetracecontainer!(trace, _NESTED_WRITE_TRACE_LIST, value, + present) + present || return + value isa shape.value_type || throw(ArgumentError( + "Parquet list field $(repr(shape.name)) contains $(typeof(value)); " * + "expected $(shape.value_type)")) + value isa AbstractVector || throw(ArgumentError( + "Parquet LIST values must be vectors")) + _nestedwritecheckcontainer(value, limits, + "Parquet LIST field $(repr(shape.name))") + count = _nestedvectorcount(value, "Parquet LIST field $(repr(shape.name))") + first, last = _nestedvectoraxes(value, + "Parquet LIST field $(repr(shape.name))") + rowsnapshot = _nestedwritelistrowsnapshot!(trace, value, shape, limits) + iszero(count) || _nestedwritestackpush!(stack, _NestedWriteScanAction( + _NESTED_WRITE_SCAN_LIST, shape, value, row_witness, nothing, nothing, + nothing, nothing, rowsnapshot, nothing, first, count, first, last), + budget) + return +end + +function _nestedwritescandictrelease!( + stack::_NestedWritePassStack{_NestedWriteScanAction}, + shape::_NestedWriteMapShape, + materialization::_NestedWriteDictMaterialization, trace, + budget::_LiveByteBudget) + action = _NestedWriteScanAction(_NESTED_WRITE_SCAN_DICT_RELEASE, shape, + nothing, nothing, nothing, nothing, nothing, materialization, nothing, + nothing, 0, 0, 0, 0) + try + _nestedwritestackpush!(stack, action, budget) + catch + _nestedwritedictrelease!(trace, materialization) + rethrow() + end + return +end + +function _nestedwritescanmap!( + stack::_NestedWritePassStack{_NestedWriteScanAction}, + shape::_NestedWriteMapShape, value, limits::Limits, trace, row_witness, + budget::_LiveByteBudget) + _nestedwriterowcheck(row_witness) + value isa MapValue && _validatemapvalue(value) + present = _nestedwritepresent(shape, value) + if !present + _nestedwritetracecontainer!(trace, _NESTED_WRITE_TRACE_MAP, value, + false) + return + end + value isa shape.value_type || throw(ArgumentError( + "Parquet map field $(repr(shape.name)) contains $(typeof(value)); " * + "expected $(shape.value_type)")) + if value isa AbstractDict + materialization = _nestedwritedictmaterialize(trace, value, + shape.key, shape.value, limits) + _nestedwritescandictrelease!(stack, shape, materialization, trace, + budget) + count = materialization.count + _nestedwritetracedict!(trace, count) + if !iszero(count) + _nestedwritestackpush!(stack, _NestedWriteScanAction( + _NESTED_WRITE_SCAN_MAP_DICT, shape, value, row_witness, + nothing, materialization.first, nothing, materialization, + nothing, nothing, 0, count, 0, 0), budget) + end + return + end + _nestedwritetracecontainer!(trace, _NESTED_WRITE_TRACE_MAP, value, true) + count = value isa AbstractVector ? + _nestedvectorcount(value, "Parquet MAP field $(repr(shape.name))") : 0 + _checklimit(:container_elements, count, + limits.max_container_elements) + first, last = value isa AbstractVector ? + _nestedvectoraxes(value, "Parquet MAP field $(repr(shape.name))") : + (0, 0) + iszero(count) && return + value = value::MapValue + keysnapshot = _nestedwriteoccurrencesnapshot!(trace, value.keys, + shape.key, limits) + valuesnapshot = shape.source_has_values ? + _nestedwriteoccurrencesnapshot!(trace, + something(value.values), shape.value, limits) : nothing + _nestedwritestackpush!(stack, _NestedWriteScanAction( + _NESTED_WRITE_SCAN_MAP_VIEW_KEY, shape, value, row_witness, + nothing, nothing, nothing, nothing, keysnapshot, valuesnapshot, 1, + count, first, last), budget) + return +end + +function _nestedwritescanprocess!( + stack::_NestedWritePassStack{_NestedWriteScanAction}, + action::_NestedWriteScanAction, limits::Limits, trace, + budget::_LiveByteBudget) + kind = action.kind + shape = action.shape + if kind == _NESTED_WRITE_SCAN_ENTER + shape isa _NestedWriteLeafShape && return _nestedwritescanleaf!(shape, + action.value, trace, action.row_witness) + shape isa _NestedWriteStructShape && return _nestedwritescanstruct!( + stack, shape, action.value, trace, action.row_witness, budget) + shape isa _NestedWriteListShape && return _nestedwritescanlist!(stack, + shape, action.value, limits, trace, action.row_witness, budget) + return _nestedwritescanmap!(stack, shape::_NestedWriteMapShape, + action.value, limits, trace, action.row_witness, budget) + elseif kind == _NESTED_WRITE_SCAN_POSTCHECK + _nestedwriterowcheck(action.row_witness) + _nestedwriterowcheck(action.other_witness) + return + elseif kind == _NESTED_WRITE_SCAN_KEYASSERT + _nestedwritekeyassert!(action.expected, action.value, shape, limits, + trace, nothing, action.row_witness) + return + elseif kind == _NESTED_WRITE_SCAN_DICT_RELEASE + _nestedwritedictrelease!(trace, something(action.materialization)) + return + elseif kind == _NESTED_WRITE_SCAN_STRUCT + structshape = shape::_NestedWriteStructShape + index = action.position + if structshape.package_owned + value = action.value::StructValue + child = _nestedwritestructaccesschild(value, value.names, + value.children, action.count, index, structshape.names[index], + something(structshape.source_children)[index]) + snapshot = _nestedwriteoccurrencesnapshot!(trace, child, + structshape.children[index], limits) + item, child_witness = _nestedwriterowaccess(child, value.index, + snapshot) + if index < action.count + _nestedwritestackpush!(stack, _NestedWriteScanAction(kind, + shape, action.value, action.row_witness, nothing, nothing, + nothing, nothing, nothing, nothing, index + 1, + action.count, 0, 0), budget) + end + _nestedwritestackpush!(stack, _nestedwritescanpost(shape, + child_witness, action.row_witness), budget) + _nestedwritestackpush!(stack, _nestedwritescanenter( + structshape.children[index], item, child_witness), budget) + else + item = getfield(action.value, index) + if index < action.count + _nestedwritestackpush!(stack, _NestedWriteScanAction(kind, + shape, action.value, action.row_witness, nothing, nothing, + nothing, nothing, nothing, nothing, index + 1, + action.count, 0, 0), budget) + end + _nestedwritestackpush!(stack, _nestedwritescanenter( + structshape.children[index], item, nothing), budget) + end + return + elseif kind == _NESTED_WRITE_SCAN_LIST + listshape = shape::_NestedWriteListShape + index = action.position + _nestedwriterowcheck(action.row_witness) + _nestedwriteviewaccesscheck(action.value, action.count, action.first, + action.last) + item, child_witness = _nestedwritelistrowaccess(action.value, index, + action.snapshot1) + _nestedwriteviewaccesscheck(action.value, action.count, action.first, + action.last) + index < action.last && _nestedwritestackpush!(stack, + _NestedWriteScanAction(kind, shape, action.value, + action.row_witness, nothing, nothing, nothing, nothing, + action.snapshot1, nothing, index + 1, action.count, + action.first, action.last), budget) + _nestedwritestackpush!(stack, _nestedwritescanpost(shape, + child_witness, action.row_witness), budget) + _nestedwritestackpush!(stack, _nestedwritescanenter(listshape.element, + item, child_witness), budget) + return + elseif kind == _NESTED_WRITE_SCAN_MAP_DICT + entry = action.state + if action.position == 1 + entry = entry.next + entry === nothing && return + end + mapshape = shape::_NestedWriteMapShape + ismissing(entry.key) && throw(ArgumentError( + "Parquet MAP field $(repr(mapshape.name)) contains a missing key")) + expected = _nestedwritetracekey!(trace, entry.key, mapshape.key, + limits) + _nestedwritestackpush!(stack, _NestedWriteScanAction(kind, shape, + action.value, action.row_witness, nothing, entry, nothing, + action.materialization, nothing, nothing, 1, action.count, 0, 0), + budget) + mapvalue = mapshape.source_has_values ? entry.value : missing + _nestedwritestackpush!(stack, _nestedwritescanenter(mapshape.value, + mapvalue, nothing), budget) + _nestedwritestackpush!(stack, _NestedWriteScanAction( + _NESTED_WRITE_SCAN_KEYASSERT, mapshape.key, entry.key, nothing, + nothing, nothing, expected, nothing, nothing, nothing, 0, 0, 0, + 0), budget) + _nestedwritestackpush!(stack, _nestedwritescanenter(mapshape.key, + entry.key, nothing), budget) + return + elseif kind == _NESTED_WRITE_SCAN_MAP_VIEW_KEY + mapshape = shape::_NestedWriteMapShape + entry = action.position + _nestedwriterowcheck(action.row_witness) + _nestedwriteviewaccesscheck(action.value, action.count, action.first, + action.last) + physical = action.value.first + entry - 1 + keyvalue, key_witness = _nestedwriterowaccess(action.value.keys, + physical, action.snapshot1) + ismissing(keyvalue) && throw(ArgumentError( + "Parquet MAP field $(repr(mapshape.name)) contains a missing key")) + expected = _nestedwritetracekey!(trace, keyvalue, mapshape.key, limits, + nothing, key_witness) + _nestedwritestackpush!(stack, _NestedWriteScanAction( + _NESTED_WRITE_SCAN_MAP_VIEW_VALUE, shape, action.value, + action.row_witness, key_witness, keyvalue, expected, nothing, + action.snapshot1, action.snapshot2, physical, action.count, + action.first, action.last), budget) + _nestedwritestackpush!(stack, _nestedwritescanenter(mapshape.key, + keyvalue, key_witness), budget) + return + elseif kind == _NESTED_WRITE_SCAN_MAP_VIEW_VALUE + mapshape = shape::_NestedWriteMapShape + keyvalue = action.state + _nestedwritekeyassert!(action.expected, keyvalue, mapshape.key, limits, + trace, nothing, action.other_witness) + _nestedwriterowcheck(action.other_witness) + _nestedwriterowcheck(action.row_witness) + if mapshape.source_has_values + mapvalue, value_witness = _nestedwriterowaccess( + something(action.value.values), action.position, + action.snapshot2) + else + mapvalue = missing + value_witness = nothing + end + entry = action.position - action.value.first + 1 + entry < action.count && _nestedwritestackpush!(stack, + _NestedWriteScanAction(_NESTED_WRITE_SCAN_MAP_VIEW_KEY, shape, + action.value, action.row_witness, nothing, nothing, nothing, + nothing, action.snapshot1, action.snapshot2, entry + 1, + action.count, action.first, action.last), budget) + _nestedwritestackpush!(stack, _nestedwritescanpost(shape, + value_witness, action.row_witness), budget) + _nestedwritestackpush!(stack, _nestedwritescanenter(mapshape.value, + mapvalue, value_witness), budget) + return + end + return +end + +function _nestedwritescancleanup!( + stack::_NestedWritePassStack{_NestedWriteScanAction}, trace) + for action in Iterators.reverse(stack.frames) + if action.kind == _NESTED_WRITE_SCAN_DICT_RELEASE + _nestedwritedictrelease!(trace, + something(action.materialization)) + end + end + return +end + +function _nestedwritescanaggregate!( + stack::_NestedWritePassStack{_NestedWriteScanAction}, + shape::_NestedWriteShape, value, limits::Limits, trace, row_witness, + budget::_LiveByteBudget) + Base.@nospecialize value + shape isa _NestedWriteLeafShape && return _nestedwritescanleaf!(shape, + value, trace, row_witness) + isempty(stack.frames) && !stack.processing || throw(AssertionError( + "nested writer aggregate scratch stack is not empty")) + try + _nestedwritestackpush!(stack, + _nestedwritescanenter(shape, value, row_witness), budget) + while !isempty(stack.frames) + action = _nestedwritepassstackpop!(stack) + try + _nestedwritescanprocess!(stack, action, limits, trace, budget) + finally + _nestedwritepassstackprocessed!(stack) + end + end + finally + if !isempty(stack.frames) + try + _nestedwritescancleanup!(stack, trace) + finally + _nestedwritepassstackclear!(stack) + end + end + end + isempty(stack.frames) && !stack.processing || throw(AssertionError( + "nested writer aggregate scratch stack retained actions")) + return +end + +function _nestedwritescanaggregate!(shape::_NestedWriteShape, value, + limits::Limits, trace, row_witness) + Base.@nospecialize value + shape isa _NestedWriteLeafShape && return _nestedwritescanleaf!(shape, + value, trace, row_witness) + budget = trace isa _NestedWriteTrace ? trace.budget : + _LiveByteBudget(limits) + stack = _nestedwritepassstackstart(_NestedWriteScanAction, budget) + try + _nestedwritescanaggregate!(stack, shape, value, limits, trace, + row_witness, budget) + finally + _nestedwritepassstackrelease!(stack, budget) + end + return +end + +function _nestedwritescanaggregate!(shape::_NestedWriteShape, value, + limits::Limits, trace) + return _nestedwritescanaggregate!(shape, value, limits, trace, nothing) +end + +function _nestedwritescanaggregate!(shape::_NestedWriteShape, value, + limits::Limits) + return _nestedwritescanaggregate!(shape, value, limits, nothing, nothing) +end + +function _nestedwriteintegerlogical(name::String, value_type::Type, + optional::Bool) + element = _integerwriteelement(name, value_type, optional) + element === nothing && throw(ArgumentError( + "unsupported Parquet integer type $value_type")) + return element +end + +function _nestedwritetimestampelement(shape::_NestedWriteLeafShape) + aggregate = shape.aggregate + aggregate.seen || throw(ArgumentError( + "cannot infer TIMESTAMP UTC adjustment for empty or all-null field " * + "$(repr(shape.name)); use Parquet.LogicalColumn")) + value_type = shape.value_type + adjusted = something(aggregate.adjusted) + unit = _timestampwriteunit(value_type) + logical = Metadata.LogicalType(TIMESTAMP=Metadata.TimestampType( + isAdjustedToUTC=adjusted, unit=unit)) + return _logicalwriteelement(shape.name, Metadata.Type.INT64, shape.optional; + logical=logical, + converted=_canonicalconverted( + _TimestampLogicalKind(_timestampwriteunitcode(value_type), adjusted))) +end + +function _nestedwritedecimalelement(shape::_NestedWriteLeafShape, + limits::Limits) + aggregate = shape.aggregate + aggregate.seen || throw(ArgumentError( + "cannot infer DECIMAL precision and scale for empty or all-null field " * + "$(repr(shape.name)); use Parquet.LogicalColumn")) + precision = aggregate.precision + scale = something(aggregate.scale) + if precision <= 9 + physical = Metadata.Type.INT32 + width = nothing + elseif precision <= 18 + physical = Metadata.Type.INT64 + width = nothing + else + physical = Metadata.Type.FIXED_LEN_BYTE_ARRAY + width = _decimalwritewidth(precision, limits) + end + logical = Metadata.LogicalType(DECIMAL=Metadata.DecimalType( + scale=scale, precision=precision)) + return _logicalwriteelement(shape.name, physical, shape.optional; + width=width, logical=logical, converted=Metadata.ConvertedType.DECIMAL, + scale=scale, precision=precision) +end + +function _nestedwriteleafelement(shape::_NestedWriteLeafShape, + limits::Limits) + shape.explicit === nothing || return _logicalcolumnwriteelement(shape.name, + shape.explicit, shape.optional, limits) + value_type = shape.value_type + value_type === Missing && return _logicalwriteelement(shape.name, + Metadata.Type.INT32, true; + logical=Metadata.LogicalType(UNKNOWN=Metadata.NullType())) + shape.fixed_width === nothing || return _logicalwriteelement(shape.name, + Metadata.Type.FIXED_LEN_BYTE_ARRAY, shape.optional; + width=shape.fixed_width) + value_type == Dates.Date && return _logicalwriteelement(shape.name, + Metadata.Type.INT32, shape.optional; + logical=Metadata.LogicalType(DATE=Metadata.DateType()), + converted=Metadata.ConvertedType.DATE) + value_type in (Int8, Int16, UInt8, UInt16, UInt32, UInt64) && + return _nestedwriteintegerlogical(shape.name, value_type, shape.optional) + if value_type == Dates.Time + unit = Metadata.TimeUnit(NANOS=Metadata.NanoSeconds()) + logical = Metadata.LogicalType(TIME=Metadata.TimeType( + isAdjustedToUTC=false, unit=unit)) + return _logicalwriteelement(shape.name, Metadata.Type.INT64, + shape.optional; logical=logical) + elseif value_type == Dates.DateTime + unit = Metadata.TimeUnit(MILLIS=Metadata.MilliSeconds()) + logical = Metadata.LogicalType(TIMESTAMP=Metadata.TimestampType( + isAdjustedToUTC=false, unit=unit)) + return _logicalwriteelement(shape.name, Metadata.Type.INT64, + shape.optional; logical=logical, + converted=Metadata.ConvertedType.TIMESTAMP_MILLIS) + elseif value_type <: Timestamp + return _nestedwritetimestampelement(shape) + elseif value_type == Decimal + return _nestedwritedecimalelement(shape, limits) + end + binary = _binarywriteelement(shape.name, value_type, shape.optional) + binary === nothing || return binary + physical = _writetype(value_type) + width = _writetypelength(value_type) + logical, converted = _stringlogical(value_type) + return _logicalwriteelement(shape.name, physical, shape.optional; + width=width, logical=logical, converted=converted) +end + +function _nestedwriteschemastart(shape::_NestedWriteShape, limits::Limits, + budget::_LiveByteBudget) + if shape isa _NestedWriteLeafShape + _reservearray!(budget, Metadata.SchemaElement, 1) + return Metadata.SchemaElement[_nestedwriteleafelement(shape, limits)] + elseif shape isa _NestedWriteStructShape + length(shape.children) <= typemax(Int32) || throw(ArgumentError( + "Parquet struct has more than Int32 children")) + _reservearray!(budget, Vector{Metadata.SchemaElement}, + length(shape.children)) + fragments = Vector{Vector{Metadata.SchemaElement}}(undef, + length(shape.children)) + return _NestedWriteSchemaFrame(shape, fragments, nothing, nothing, 0, 1) + end + return _NestedWriteSchemaFrame(shape, nothing, nothing, nothing, 0, 0) +end + +function _nestedwriteschemaexpected(frame::_NestedWriteSchemaFrame) + shape = frame.shape + shape isa _NestedWriteStructShape && return length(shape.children) + shape isa _NestedWriteListShape && return 1 + return 2 +end + +function _nestedwriteschemaaccept(frame::_NestedWriteSchemaFrame, + fragment::Vector{Metadata.SchemaElement}, limits::Limits) + shape = frame.shape + first = frame.first + second = frame.second + count = frame.count + if shape isa _NestedWriteStructShape + something(frame.fragments)[frame.position + 1] = fragment + count = Base.checked_add(count, length(fragment)) + _checklimit(:container_elements, count, + limits.max_container_elements) + elseif frame.position == 0 + first = fragment + if shape isa _NestedWriteListShape + count = Base.checked_add(2, length(fragment)) + _checklimit(:container_elements, count, + limits.max_container_elements) + end + else + second = fragment + count = Base.checked_add(2, + Base.checked_add(length(something(first)), length(fragment))) + _checklimit(:container_elements, count, + limits.max_container_elements) + end + return _NestedWriteSchemaFrame(shape, frame.fragments, first, second, + frame.position + 1, count) +end + +function _nestedwriteschemanext(frame::_NestedWriteSchemaFrame) + shape = frame.shape + shape isa _NestedWriteStructShape && + return shape.children[frame.position + 1] + shape isa _NestedWriteListShape && return shape.element + mapshape = shape::_NestedWriteMapShape + return frame.position == 0 ? mapshape.key : mapshape.value +end + +function _nestedwriteschemafinish(frame::_NestedWriteSchemaFrame, + budget::_LiveByteBudget) + shape = frame.shape + _reservearray!(budget, Metadata.SchemaElement, frame.count) + repetition = getfield(shape, :optional) ? + Metadata.FieldRepetitionType.OPTIONAL : + Metadata.FieldRepetitionType.REQUIRED + if shape isa _NestedWriteStructShape + fragments = something(frame.fragments) + output = Metadata.SchemaElement[Metadata.SchemaElement( + repetition_type=repetition, name=shape.name, + num_children=Int32(length(shape.children)))] + sizehint!(output, frame.count) + for fragment in fragments + append!(output, fragment) + _release!(budget, _materializedarraybytes(Metadata.SchemaElement, + length(fragment))) + end + _release!(budget, _materializedarraybytes( + Vector{Metadata.SchemaElement}, length(fragments))) + return output + elseif shape isa _NestedWriteListShape + outer = Metadata.SchemaElement(repetition_type=repetition, + name=shape.name, num_children=Int32(1), + converted_type=Metadata.ConvertedType.LIST, + logicalType=Metadata.LogicalType(LIST=Metadata.ListType())) + repeated = Metadata.SchemaElement( + repetition_type=Metadata.FieldRepetitionType.REPEATED, + name="list", num_children=Int32(1)) + element = something(frame.first) + output = Metadata.SchemaElement[outer, repeated] + sizehint!(output, frame.count) + append!(output, element) + _release!(budget, _materializedarraybytes(Metadata.SchemaElement, + length(element))) + return output + end + mapshape = shape::_NestedWriteMapShape + outer = Metadata.SchemaElement(repetition_type=repetition, + name=mapshape.name, num_children=Int32(1), + converted_type=Metadata.ConvertedType.MAP, + logicalType=Metadata.LogicalType(MAP=Metadata.MapType())) + repeated = Metadata.SchemaElement( + repetition_type=Metadata.FieldRepetitionType.REPEATED, + name="key_value", num_children=Int32(2)) + key = something(frame.first) + value = something(frame.second) + output = Metadata.SchemaElement[outer, repeated] + sizehint!(output, frame.count) + append!(output, key) + append!(output, value) + _release!(budget, _materializedarraybytes(Metadata.SchemaElement, + length(key))) + _release!(budget, _materializedarraybytes(Metadata.SchemaElement, + length(value))) + return output +end + +function _nestedwriteschema(shape::_NestedWriteShape, limits::Limits, + budget::_LiveByteBudget) + stack = _nestedwritepassstackstart(_NestedWriteSchemaFrame, budget) + pending::Union{Nothing,Vector{Metadata.SchemaElement}} = nothing + try + started = _nestedwriteschemastart(shape, limits, budget) + started isa Vector{Metadata.SchemaElement} && return started + _nestedwritestackpush!(stack, started::_NestedWriteSchemaFrame, budget) + while true + if pending !== nothing + stack[end] = _nestedwriteschemaaccept(stack[end], pending, + limits) + pending = nothing + end + frame = stack[end] + if frame.position == _nestedwriteschemaexpected(frame) + pending = _nestedwriteschemafinish(frame, budget) + _nestedwritestackpop!(stack, budget) + isempty(stack) && return pending + continue + end + started = _nestedwriteschemastart(_nestedwriteschemanext(frame), + limits, budget) + if started isa Vector{Metadata.SchemaElement} + pending = started + else + _nestedwritestackpush!(stack, + started::_NestedWriteSchemaFrame, budget) + end + end + finally + _nestedwritepassstackrelease!(stack, budget) + end +end + +function _nestedwritebindstart(shape::_NestedWriteShape, + semantic::_NestedPlan, budget::_LiveByteBudget) + if shape isa _NestedWriteLeafShape + semantic isa _NestedLeafPlan || throw(AssertionError( + "canonical writer leaf did not compile as a semantic leaf")) + _reserveobjects!(budget) + return _NestedWriteLeafPlan(semantic, shape) + elseif shape isa _NestedWriteStructShape + semantic isa _NestedStructPlan || throw(AssertionError( + "canonical writer struct did not compile as a semantic struct")) + length(shape.children) == length(semantic.children) || throw( + AssertionError( + "canonical writer struct child count changed during schema compilation")) + _reservearray!(budget, _NestedWriteNodePlan, length(shape.children)) + children = _NestedWriteNodePlan[] + sizehint!(children, length(shape.children)) + return _NestedWriteBindFrame(shape, semantic, children, nothing, + nothing, 0) + elseif shape isa _NestedWriteListShape + semantic isa _NestedListPlan || throw(AssertionError( + "canonical writer LIST did not compile as a semantic list")) + semantic.annotation == :modern_list || throw(AssertionError( + "canonical writer LIST lost its modern annotation")) + semantic.entry.element.name == "list" || throw(AssertionError( + "canonical writer LIST has a noncanonical repeated wrapper")) + return _NestedWriteBindFrame(shape, semantic, nothing, nothing, + nothing, 0) + end + mapshape = shape::_NestedWriteMapShape + semantic isa _NestedMapPlan || throw(AssertionError( + "canonical writer MAP did not compile as a semantic map")) + semantic.annotation == :modern_map || throw(AssertionError( + "canonical writer MAP lost its modern annotation")) + semantic.entry.element.name == "key_value" || throw(AssertionError( + "canonical writer MAP has a noncanonical repeated entry")) + semantic.optional_key && throw(AssertionError( + "canonical writer MAP compiled an optional key")) + semantic.value === nothing && throw(AssertionError( + "canonical writer MAP omitted its value field")) + return _NestedWriteBindFrame(mapshape, semantic, nothing, nothing, + nothing, 0) +end + +function _nestedwritebindexpected(frame::_NestedWriteBindFrame) + shape = frame.shape + shape isa _NestedWriteStructShape && return length(shape.children) + shape isa _NestedWriteListShape && return 1 + return 2 +end + +function _nestedwritebindaccept(frame::_NestedWriteBindFrame, + plan::_NestedWriteNodePlan) + first = frame.first + second = frame.second + if frame.shape isa _NestedWriteStructShape + push!(something(frame.children), plan) + elseif frame.position == 0 + first = plan + else + second = plan + end + return _NestedWriteBindFrame(frame.shape, frame.semantic, frame.children, + first, second, frame.position + 1) +end + +function _nestedwritebindnext(frame::_NestedWriteBindFrame) + shape = frame.shape + semantic = frame.semantic + if shape isa _NestedWriteStructShape + structsemantic = semantic::_NestedStructPlan + index = frame.position + 1 + return shape.children[index], structsemantic.children[index] + elseif shape isa _NestedWriteListShape + return shape.element, (semantic::_NestedListPlan).element + end + mapshape = shape::_NestedWriteMapShape + mapsemantic = semantic::_NestedMapPlan + frame.position == 0 && return mapshape.key, mapsemantic.key + return mapshape.value, something(mapsemantic.value) +end + +function _nestedwritebindfinish(frame::_NestedWriteBindFrame, + budget::_LiveByteBudget) + _reserveobjects!(budget) + shape = frame.shape + semantic = frame.semantic + shape isa _NestedWriteStructShape && return _NestedWriteStructPlan( + semantic::_NestedStructPlan, shape, something(frame.children)) + shape isa _NestedWriteListShape && return _NestedWriteListPlan( + semantic::_NestedListPlan, shape, something(frame.first)) + return _NestedWriteMapPlan(semantic::_NestedMapPlan, + shape::_NestedWriteMapShape, something(frame.first), + something(frame.second)) +end + +function _nestedwritebind(shape::_NestedWriteShape, + semantic::_NestedPlan, budget::_LiveByteBudget) + stack = _nestedwritepassstackstart(_NestedWriteBindFrame, budget) + pending::Union{Nothing,_NestedWriteNodePlan} = nothing + try + started = _nestedwritebindstart(shape, semantic, budget) + started isa _NestedWriteNodePlan && return started + _nestedwritestackpush!(stack, started::_NestedWriteBindFrame, budget) + while true + if pending !== nothing + stack[end] = _nestedwritebindaccept(stack[end], pending) + pending = nothing + end + frame = stack[end] + if frame.position == _nestedwritebindexpected(frame) + pending = _nestedwritebindfinish(frame, budget) + _nestedwritestackpop!(stack, budget) + isempty(stack) && return pending + continue + end + childshape, childsemantic = _nestedwritebindnext(frame) + started = _nestedwritebindstart(childshape, childsemantic, budget) + if started isa _NestedWriteNodePlan + pending = started + else + _nestedwritestackpush!(stack, + started::_NestedWriteBindFrame, budget) + end + end + finally + _nestedwritepassstackrelease!(stack, budget) + end +end + +function _nestedwritefixedpayload(element::Metadata.SchemaElement, value, + limits::Limits) + width = element.type_length + width === nothing && throw(ArgumentError( + "fixed byte-array field $(repr(element.name)) has no width")) + _checklimit(:string_bytes, width, limits.max_string_bytes) + length(value) == width || throw(ArgumentError( + "fixed byte-array field $(repr(element.name)) has a value with " * + "the wrong width")) + for byte in value + byte isa UInt8 || throw(ArgumentError( + "fixed byte-array field $(repr(element.name)) contains a non-byte value")) + end + return Int64(width) +end + +function _nestedwritedecimalpayload(element::Metadata.SchemaElement, value, + limits::Limits) + value isa Decimal || throw(ArgumentError( + "DECIMAL field $(repr(element.name)) contains a non-Decimal value")) + precision, scale = something(_decimalparameters(element)) + value.scale == scale || throw(ArgumentError( + "DECIMAL field $(repr(element.name)) requires scale $scale, got " * + "$(value.scale)")) + _checkdecimalvalue(value.unscaled, precision, element.name, ArgumentError) + physical = element.type_ + if physical == Metadata.Type.INT32 + typemin(Int32) <= value.unscaled <= typemax(Int32) || throw(ArgumentError( + "DECIMAL value does not fit in INT32")) + return Int64(0) + elseif physical == Metadata.Type.INT64 + typemin(Int64) <= value.unscaled <= typemax(Int64) || throw(ArgumentError( + "DECIMAL value does not fit in INT64")) + return Int64(0) + end + width = physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY ? + Int(element.type_length) : _twoscomplementwidth(value.unscaled) + _checklimit(:decimal_bytes, width, limits.max_decimal_bytes) + _checklimit(:string_bytes, width, limits.max_string_bytes) + return Int64(width) +end + +function _nestedwritebinarypayload(kind::Symbol, + element::Metadata.SchemaElement, value, limits::Limits) + if kind === :enum + value isa AbstractString || throw(ArgumentError( + "ENUM field $(repr(element.name)) contains a non-string value")) + bytes = codeunits(value) + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + isvalid(String, bytes) || throw(ArgumentError( + "ENUM field $(repr(element.name)) contains invalid UTF-8")) + return Int64(length(bytes)) + elseif kind === :uuid + value isa UUIDs.UUID || throw(ArgumentError( + "UUID field $(repr(element.name)) contains a non-UUID value")) + return Int64(16) + elseif kind === :float16 + value isa Float16 || throw(ArgumentError( + "FLOAT16 field $(repr(element.name)) contains a non-Float16 value")) + return Int64(2) + elseif kind === :json + value isa JSONValue || throw(ArgumentError( + "JSON field $(repr(element.name)) contains an untagged value")) + _validatejson(value.bytes, limits, ArgumentError) + return Int64(length(value.bytes)) + elseif kind === :bson + value isa BSONValue || throw(ArgumentError( + "BSON field $(repr(element.name)) contains an untagged value")) + _validatebson(value.bytes, limits, ArgumentError) + return Int64(length(value.bytes)) + elseif kind === :interval + value isa Interval || throw(ArgumentError( + "INTERVAL field $(repr(element.name)) contains a non-Interval value")) + return Int64(12) + elseif kind === :unknown + throw(ArgumentError( + "UNKNOWN field $(repr(element.name)) can contain only missing")) + end + throw(ArgumentError("unsupported binary logical kind $kind")) +end + +function _nestedwriteleafpayload(element::Metadata.SchemaElement, value, + limits::Limits) + kind = _logicalkind(element) + if kind isa Union{_TimeLogicalKind,_TimestampLogicalKind, + _IntegerLogicalKind} + _temporalphysicalvalue(kind, element, value) + return Int64(0) + elseif kind === :string + value isa AbstractString || throw(ArgumentError( + "STRING field $(repr(element.name)) contains a non-string value")) + bytes = codeunits(value) + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + isvalid(String, bytes) || throw(ArgumentError( + "STRING field $(repr(element.name)) contains invalid UTF-8")) + return Int64(length(bytes)) + elseif kind === :date + value isa Dates.Date || throw(ArgumentError( + "DATE field $(repr(element.name)) contains a non-Date value")) + _toparquetdate(value) + return Int64(0) + elseif kind === :decimal + return _nestedwritedecimalpayload(element, value, limits) + elseif kind isa Symbol + return _nestedwritebinarypayload(kind, element, value, limits) + end + physical = element.type_ + expected = _physicaleltype(physical) + if physical == Metadata.Type.BYTE_ARRAY + value isa AbstractVector{UInt8} || throw(ArgumentError( + "BYTE_ARRAY field $(repr(element.name)) contains a non-byte-array value")) + _checklimit(:string_bytes, length(value), limits.max_string_bytes) + return Int64(length(value)) + elseif physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY + value isa Union{Tuple,AbstractVector} || throw(ArgumentError( + "fixed byte-array field $(repr(element.name)) contains $(typeof(value))")) + return _nestedwritefixedpayload(element, value, limits) + end + value isa expected || throw(ArgumentError( + "physical field $(repr(element.name)) contains $(typeof(value)); " * + "expected $expected")) + return Int64(0) +end + +function _nestedwritenormalizephysical(element::Metadata.SchemaElement, value, + limits::Limits) + physical = _physicalvalue(element, value; limits=limits) + if element.type_ in (Metadata.Type.BYTE_ARRAY, + Metadata.Type.FIXED_LEN_BYTE_ARRAY) + physical isa Vector{UInt8} && physical !== value && return physical + bytes = physical isa AbstractString ? codeunits(physical) : physical + return UInt8[byte for byte in bytes] + end + return physical +end + +function _nestedwriteincrement(value::Int64, limits::Limits) + next = try + Base.checked_add(value, Int64(1)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, next, + limits.max_container_elements) + return next +end + +function _nestedwriteaddpayload(value::Int64, bytes::Int64, limits::Limits) + requested = try + Base.checked_add(value, bytes) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:materialized_bytes, typemax(Int64), + limits.max_materialized_bytes)) + end + _checklimit(:materialized_bytes, requested, + limits.max_materialized_bytes) + return requested +end + +function _nestedwriterecord!(context::_NestedWriteCountContext, + leafindex::Int, repetition::UInt64, definition::UInt64, + payload::Int64, present::Bool, value) + count = context.counts[leafindex] + count.entries = _nestedwriteincrement(count.entries, context.limits) + if present + count.dense = _nestedwriteincrement(count.dense, context.limits) + count.payload_bytes = _nestedwriteaddpayload(count.payload_bytes, + payload, context.limits) + end + return +end + +function _nestedwriterecord!(context::_NestedWriteEmitContext, + leafindex::Int, repetition::UInt64, definition::UInt64, + payload::Int64, present::Bool, value) + builder = context.builders[leafindex] + entry = builder.entry_position + 1 + entry <= length(builder.repetition) || throw(ArgumentError( + "nested Parquet input changed its level-entry count between passes")) + builder.entry_position = entry + builder.repetition[entry] = repetition + builder.definition[entry] = definition + if present + dense = builder.dense_position + 1 + dense <= length(builder.values) || throw(ArgumentError( + "nested Parquet input changed its dense-value count between passes")) + builder.dense_position = dense + builder.payload_position = _nestedwriteaddpayload( + builder.payload_position, payload, context.limits) + builder.values[dense] = value + end + return +end + +function _nestedwritepreflightemit(context::_NestedWriteEmitContext, + leafindex::Int, payload::Int64) + builder = context.builders[leafindex] + count = context.counts[leafindex] + builder.entry_position < count.entries || throw(ArgumentError( + "nested Parquet input changed its level-entry count between passes")) + builder.dense_position < count.dense || throw(ArgumentError( + "nested Parquet input changed its dense-value count between passes")) + requested = try + Base.checked_add(builder.payload_position, payload) + catch err + err isa OverflowError || rethrow() + throw(ArgumentError( + "nested Parquet input changed its variable-width payload between passes")) + end + requested <= count.payload_bytes || throw(ArgumentError( + "nested Parquet input changed its variable-width payload between passes")) + return +end + +function _nestedwritemarker!(context, range::UnitRange{Int32}, + repetition::UInt64, definition::UInt64) + for rawindex in range + _nestedwriterecord!(context, Int(rawindex), repetition, definition, + Int64(0), false, nothing) + end + return +end + +function _nestedwriteshred!(context::_NestedWriteCountContext, + plan::_NestedWriteLeafPlan, value, repetition::UInt64) + return _nestedwriteshred!(context, plan, value, repetition, nothing) +end + +function _nestedwriteshred!(context::_NestedWriteCountContext, + plan::_NestedWriteLeafPlan, value, repetition::UInt64, row_witness) + _nestedwriterowcheck(row_witness) + semantic = plan.semantic + shape = plan.shape + present = _nestedwritepresent(shape, value) + _nestedwritetraceleaf!(context.trace, value, present) + if !present + _nestedwriterecord!(context, Int(first(semantic.leaf_range)), repetition, + UInt64(semantic.parent_definition), Int64(0), false, nothing) + return + end + _nestedwritecheckvalue(shape, value) + payload = _nestedwriteleafpayload(semantic.source.element, value, + context.limits) + _nestedwriterecord!(context, Int(first(semantic.leaf_range)), repetition, + UInt64(semantic.present_definition), payload, true, value) + return +end + +function _nestedwriteshred!(context::_NestedWriteEmitContext, + plan::_NestedWriteLeafPlan, value, repetition::UInt64) + return _nestedwriteshred!(context, plan, value, repetition, nothing) +end + +function _nestedwriteshred!(context::_NestedWriteEmitContext, + plan::_NestedWriteLeafPlan, value, repetition::UInt64, row_witness) + _nestedwriterowcheck(row_witness) + semantic = plan.semantic + shape = plan.shape + present = _nestedwritepresent(shape, value) + _nestedwritetraceleaf!(context.trace, value, present) + if !present + _nestedwriterecord!(context, Int(first(semantic.leaf_range)), repetition, + UInt64(semantic.parent_definition), Int64(0), false, nothing) + return + end + _nestedwritecheckvalue(shape, value) + payload = _nestedwriteleafpayload(semantic.source.element, value, + context.limits) + leafindex = Int(first(semantic.leaf_range)) + _nestedwritepreflightemit(context, leafindex, payload) + physical = _nestedwritenormalizephysical(semantic.source.element, value, + context.limits) + _nestedwriterecord!(context, leafindex, repetition, + UInt64(semantic.present_definition), payload, true, physical) + return +end + +function _nestedwritekeymismatch() + throw(ArgumentError( + "nested Parquet MAP key changed while it was physically consumed")) +end + +function _nestedwritekeymissing(expected::_NestedWriteKeySnapshot) + return expected.kind == _NESTED_WRITE_KEY_SCALAR && + expected.source_type === Missing && ismissing(expected.value) +end + +function _nestedwritekeyleaflogicalequal(expected::_NestedWriteKeySnapshot, + value) + if expected.kind == _NESTED_WRITE_KEY_SCALAR + return typeof(value) === expected.source_type && + isequal(value, expected.value) + elseif expected.kind == _NESTED_WRITE_KEY_STRING + return value isa AbstractString && + typeof(value) === expected.source_type + elseif expected.kind == _NESTED_WRITE_KEY_BYTES + if expected.source_type === JSONValue + return value isa JSONValue + elseif expected.source_type === BSONValue + return value isa BSONValue + end + return typeof(value) === expected.source_type + end + return false +end + +function _nestedwritekeyleafstatecheck(expected::_NestedWriteKeySnapshot, + value) + expected.kind in (_NESTED_WRITE_KEY_STRING, + _NESTED_WRITE_KEY_BYTES) || return + source = if value isa AbstractString + codeunits(value) + elseif value isa Union{JSONValue,BSONValue} + value.bytes + else + value + end + state = _nestedwritekeyindexedstate(source) + length(expected.value) == state.count && + expected.first == state.first && expected.last == state.last || throw( + ArgumentError( + "nested Parquet MAP key changed length or axes while it was consumed")) + return +end + +function _nestedwritekeysnapshotpayload(element::Metadata.SchemaElement, + expected::_NestedWriteKeySnapshot, limits::Limits) + expected.kind in (_NESTED_WRITE_KEY_STRING, + _NESTED_WRITE_KEY_BYTES) || return _nestedwriteleafpayload(element, + expected.value, limits) + bytes = expected.value::Vector{UInt8} + kind = _logicalkind(element) + if kind in (:string, :enum) + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + isvalid(String, bytes) || throw(ArgumentError( + "Parquet MAP key contains invalid UTF-8")) + return Int64(length(bytes)) + elseif kind === :json + _validatejson(bytes, limits, ArgumentError) + return Int64(length(bytes)) + elseif kind === :bson + _validatebson(bytes, limits, ArgumentError) + return Int64(length(bytes)) + elseif element.type_ == Metadata.Type.BYTE_ARRAY + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + return Int64(length(bytes)) + elseif element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY + return _nestedwritefixedpayload(element, bytes, limits) + end + throw(ArgumentError( + "Parquet MAP-key snapshot has incompatible physical bytes")) +end + +function _nestedwritekeyphysicalequal(expected::_NestedWriteKeySnapshot, + physical, element::Metadata.SchemaElement, + context::_NestedWriteEmitContext) + if expected.kind in (_NESTED_WRITE_KEY_STRING, _NESTED_WRITE_KEY_BYTES) + physical isa AbstractVector{UInt8} || return false + return _nestedwritekeybytesequal(expected.value, physical) + end + expected.source_type === Decimal || return true + trace = context.trace + trace === nothing && throw(AssertionError( + "authoritative MAP-key comparison requires a writer trace")) + charge = Int64(0) + if element.type_ in (Metadata.Type.BYTE_ARRAY, + Metadata.Type.FIXED_LEN_BYTE_ARRAY) + payload = _nestedwriteleafpayload(element, expected.value, + context.limits) + charge = _materializedarraybytes(UInt8, payload) + _nestedwritetracereserve!(trace, charge) + end + try + canonical = _nestedwritenormalizekeyphysical(element, expected.value, + context.limits) + if canonical isa AbstractVector{UInt8} + physical isa AbstractVector{UInt8} || return false + return _nestedwritekeybytesequal(canonical, physical) + end + return isequal(canonical, physical) + finally + iszero(charge) || _nestedwritetraceunreserve!(trace, charge) + end +end + +function _nestedwritekeynormalizedbytes(source, limits::Limits) + state = _nestedwritekeyindexedstate(source) + _checklimit(:string_bytes, state.count, limits.max_string_bytes) + return _nestedwritekeycopybytes(source, state, "normalized") +end + +function _nestedwritenormalizekeyphysical(element::Metadata.SchemaElement, + value, limits::Limits) + kind = _logicalkind(element) + if kind in (:string, :enum) && value isa AbstractString + bytes = _nestedwritekeynormalizedbytes(codeunits(value), limits) + isvalid(String, bytes) || throw(ArgumentError( + "STRING value contains invalid UTF-8")) + return bytes + end + physical = _physicalvalue(element, value; limits=limits) + element.type_ in (Metadata.Type.BYTE_ARRAY, + Metadata.Type.FIXED_LEN_BYTE_ARRAY) || return physical + physical isa Vector{UInt8} && physical !== value && return physical + bytes = physical isa AbstractString ? codeunits(physical) : physical + return _nestedwritekeynormalizedbytes(bytes, limits) +end + +function _nestedwriteshredkey!(context, + plan::_NestedWriteLeafPlan, value, + expected::_NestedWriteKeySnapshot, repetition::UInt64) + return _nestedwriteshredkey!(context, plan, value, expected, repetition, + nothing) +end + +function _nestedwriteshredkey!(context, + plan::_NestedWriteLeafPlan, value, + expected::_NestedWriteKeySnapshot, repetition::UInt64, row_witness) + _nestedwriterowcheck(row_witness) + semantic = plan.semantic + shape = plan.shape + present = _nestedwritepresent(shape, value) + _nestedwritetraceleaf!(context.trace, value, present) + _nestedwritekeyleafstatecheck(expected, value) + if !present + _nestedwritekeymissing(expected) || _nestedwritekeymismatch() + _nestedwriterecord!(context, Int(first(semantic.leaf_range)), + repetition, UInt64(semantic.parent_definition), Int64(0), false, + nothing) + return + end + _nestedwritecheckvalue(shape, value) + _nestedwritekeyleaflogicalequal(expected, value) || + _nestedwritekeymismatch() + element = semantic.source.element + payload = _nestedwritekeysnapshotpayload(element, expected, + context.limits) + leafindex = Int(first(semantic.leaf_range)) + physical = value + if context isa _NestedWriteEmitContext + _nestedwritepreflightemit(context, leafindex, payload) + physical = _nestedwritenormalizekeyphysical(element, value, + context.limits) + _nestedwritekeyphysicalequal(expected, physical, element, context) || + _nestedwritekeymismatch() + end + _nestedwriterecord!(context, leafindex, repetition, + UInt64(semantic.present_definition), payload, true, physical) + return +end + +const _NestedWriteContainerPlan = Union{_NestedWriteStructPlan, + _NestedWriteListPlan,_NestedWriteMapPlan} + +function _nestedwriteshredenter(plan::_NestedWriteNodePlan, value, + expected::Union{Nothing,_NestedWriteKeySnapshot}, keymode::Bool, + repetition::UInt64, row_witness) + return _NestedWriteShredAction(_NESTED_WRITE_SHRED_ENTER, plan, value, + expected, keymode, repetition, row_witness, nothing, nothing, nothing, + nothing, nothing, nothing, 0, 0, 0, 0, UInt8(0)) +end + +function _nestedwriteshredpost(plan::_NestedWriteNodePlan, first, second) + return _NestedWriteShredAction(_NESTED_WRITE_SHRED_POSTCHECK, plan, + nothing, nothing, false, UInt64(0), first, second, nothing, nothing, + nothing, nothing, nothing, 0, 0, 0, 0, UInt8(0)) +end + +function _nestedwriteshredstructenter!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, context, + plan::_NestedWriteStructPlan, value, + expected::Union{Nothing,_NestedWriteKeySnapshot}, keymode::Bool, + repetition::UInt64, row_witness, budget::_LiveByteBudget) + _nestedwriterowcheck(row_witness) + semantic = plan.semantic + shape = plan.shape + present = _nestedwritepresent(shape, value) + _nestedwritetracestruct!(context.trace, value, present, + length(plan.children)) + if !present + keymode && (_nestedwritekeymissing(something(expected)) || + _nestedwritekeymismatch()) + _nestedwritemarker!(context, semantic.leaf_range, repetition, + UInt64(semantic.parent_definition)) + return + end + if keymode + snapshot = something(expected) + expectedkind = shape.package_owned ? _NESTED_WRITE_KEY_STRUCT : + _NESTED_WRITE_KEY_NAMED_TUPLE + snapshot.kind == expectedkind || _nestedwritekeymismatch() + length(snapshot.children) == length(plan.children) || + _nestedwritekeymismatch() + if shape.package_owned + value isa StructValue || _nestedwritekeymismatch() + value.names == shape.names || _nestedwritekeymismatch() + length(value) == length(plan.children) || + _nestedwritekeymismatch() + else + value isa shape.value_type || _nestedwritekeymismatch() + end + elseif shape.package_owned + value isa StructValue || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) requires StructValue rows")) + value.names == shape.names || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) changed its field names between writer passes")) + length(value) == length(plan.children) || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) changed its field count between writer passes")) + else + value isa NamedTuple || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) requires NamedTuple rows")) + value isa shape.value_type || throw(ArgumentError( + "Parquet struct field $(repr(shape.name)) contains $(typeof(value)); " * + "expected $(shape.value_type)")) + end + _nestedwritestackpush!(stack, _NestedWriteShredAction( + _NESTED_WRITE_SHRED_STRUCT, plan, value, expected, keymode, repetition, + row_witness, nothing, nothing, nothing, nothing, nothing, nothing, 1, + length(plan.children), 0, 0, UInt8(0)), budget) + return +end + +function _nestedwriteshredlistenter!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, + context, plan::_NestedWriteListPlan, value, + expected::Union{Nothing,_NestedWriteKeySnapshot}, keymode::Bool, + repetition::UInt64, row_witness, budget::_LiveByteBudget) + _nestedwriterowcheck(row_witness) + value isa ListValue && _validatelistvalue(value) + semantic = plan.semantic + shape = plan.shape + present = _nestedwritepresent(shape, value) + _nestedwritetracecontainer!(context.trace, _NESTED_WRITE_TRACE_LIST, + value, present) + if !present + keymode && (_nestedwritekeymissing(something(expected)) || + _nestedwritekeymismatch()) + _nestedwritemarker!(context, semantic.leaf_range, repetition, + UInt64(semantic.parent_definition)) + return + end + if keymode + snapshot = something(expected) + snapshot.kind == _NESTED_WRITE_KEY_LIST || _nestedwritekeymismatch() + value isa shape.value_type || _nestedwritekeymismatch() + value isa AbstractVector || _nestedwritekeymismatch() + else + value isa shape.value_type || throw(ArgumentError( + "Parquet list field $(repr(shape.name)) contains $(typeof(value)); " * + "expected $(shape.value_type)")) + value isa AbstractVector || throw(ArgumentError( + "Parquet LIST values must be vectors")) + end + _nestedwritecheckcontainer(value, context.limits, + "Parquet LIST field $(repr(shape.name))") + label = keymode ? "nested Parquet MAP-key LIST" : + "Parquet LIST field $(repr(shape.name))" + count = _nestedvectorcount(value, label) + first, last = _nestedvectoraxes(value, label) + rowsnapshot = _nestedwritelistrowsnapshot!(context.trace, value, shape, + context.limits) + if keymode + snapshot = something(expected) + count == length(snapshot.children) || _nestedwritekeymismatch() + first == snapshot.first && last == snapshot.last || + _nestedwritekeymismatch() + end + if iszero(count) + _nestedwritemarker!(context, semantic.leaf_range, repetition, + UInt64(semantic.present_definition)) + return + end + _nestedwritestackpush!(stack, _NestedWriteShredAction( + _NESTED_WRITE_SHRED_LIST, plan, value, expected, keymode, repetition, + row_witness, nothing, nothing, nothing, nothing, rowsnapshot, nothing, + first, count, first, last, UInt8(0)), budget) + return +end + +function _nestedwriteshreddictrelease!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, context, + plan::_NestedWriteMapPlan, + materialization::_NestedWriteDictMaterialization, + budget::_LiveByteBudget) + action = _NestedWriteShredAction(_NESTED_WRITE_SHRED_DICT_RELEASE, plan, + nothing, nothing, false, UInt64(0), nothing, nothing, nothing, nothing, + materialization, nothing, nothing, 0, 0, 0, 0, UInt8(0)) + try + _nestedwritestackpush!(stack, action, budget) + catch + _nestedwritedictrelease!(context.trace, materialization) + rethrow() + end + return +end + +function _nestedwriteshredmapenter!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, + context, plan::_NestedWriteMapPlan, value, + expected::Union{Nothing,_NestedWriteKeySnapshot}, keymode::Bool, + repetition::UInt64, row_witness, budget::_LiveByteBudget) + _nestedwriterowcheck(row_witness) + value isa MapValue && _validatemapvalue(value) + semantic = plan.semantic + shape = plan.shape + present = _nestedwritepresent(shape, value) + if !present + _nestedwritetracecontainer!(context.trace, _NESTED_WRITE_TRACE_MAP, + value, false) + keymode && (_nestedwritekeymissing(something(expected)) || + _nestedwritekeymismatch()) + _nestedwritemarker!(context, semantic.leaf_range, repetition, + UInt64(semantic.parent_definition)) + return + end + if keymode + snapshot = something(expected) + snapshot.kind == _NESTED_WRITE_KEY_MAP || _nestedwritekeymismatch() + value isa shape.value_type || _nestedwritekeymismatch() + else + value isa shape.value_type || throw(ArgumentError( + "Parquet map field $(repr(shape.name)) contains $(typeof(value)); " * + "expected $(shape.value_type)")) + end + if value isa AbstractDict + materialization = _nestedwritedictmaterialize(context.trace, value, + shape.key, shape.value, context.limits) + _nestedwriteshreddictrelease!(stack, context, plan, materialization, + budget) + count = materialization.count + _nestedwritetracedict!(context.trace, count) + if keymode + children = something(expected).children + iseven(length(children)) && count == length(children) ÷ 2 || + _nestedwritekeymismatch() + end + if iszero(count) + _nestedwritemarker!(context, semantic.leaf_range, repetition, + UInt64(semantic.present_definition)) + else + _nestedwritestackpush!(stack, _NestedWriteShredAction( + _NESTED_WRITE_SHRED_MAP_DICT, plan, value, expected, keymode, + repetition, row_witness, nothing, materialization.first, + nothing, materialization, nothing, nothing, 1, count, 0, 0, + UInt8(0)), budget) + end + return + end + _nestedwritetracecontainer!(context.trace, _NESTED_WRITE_TRACE_MAP, + value, true) + label = keymode ? "nested Parquet MAP-key MAP" : + "Parquet MAP field $(repr(shape.name))" + count = value isa AbstractVector ? _nestedvectorcount(value, label) : 0 + _checklimit(:container_elements, count, + context.limits.max_container_elements) + if keymode + children = something(expected).children + iseven(length(children)) && count == length(children) ÷ 2 || + _nestedwritekeymismatch() + end + if iszero(count) + _nestedwritemarker!(context, semantic.leaf_range, repetition, + UInt64(semantic.present_definition)) + return + end + first, last = value isa AbstractVector ? + _nestedvectoraxes(value, label) : (0, 0) + value = value::MapValue + keysnapshot = _nestedwriteoccurrencesnapshot!(context.trace, + value.keys, plan.key.shape, context.limits) + valuesnapshot = shape.source_has_values ? + _nestedwriteoccurrencesnapshot!(context.trace, + something(value.values), plan.value.shape, context.limits) : + nothing + _nestedwritestackpush!(stack, _NestedWriteShredAction( + _NESTED_WRITE_SHRED_MAP_VIEW_KEY, plan, value, expected, keymode, + repetition, row_witness, nothing, nothing, nothing, nothing, + keysnapshot, valuesnapshot, 1, count, first, last, UInt8(0)), + budget) + return +end + +function _nestedwriteshredchildexpected(action::_NestedWriteShredAction, + index::Int) + action.keymode || return nothing + return something(action.expected).children[index] +end + +function _nestedwriteshredkeymode(action::_NestedWriteShredAction, + nested::Union{Nothing,_NestedWriteKeySnapshot}) + return action.keymode || nested !== nothing +end + +function _nestedwriteshredmapkeyexpected( + action::_NestedWriteShredAction, entry::Int, + nested::Union{Nothing,_NestedWriteKeySnapshot}) + action.keymode && return _nestedwriteshredchildexpected(action, + 2 * entry - 1) + return nested +end + +function _nestedwriteshredmapvalueexpected( + action::_NestedWriteShredAction, entry::Int) + action.keymode || return nothing + return _nestedwriteshredchildexpected(action, 2 * entry) +end + +function _nestedwriteshredmaprepetition(action::_NestedWriteShredAction, + entry::Int) + entry == 1 && return action.repetition + return UInt64((action.plan::_NestedWriteMapPlan).semantic.repetition_level) +end + +function _nestedwriteshredprocessstruct!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, context, + action::_NestedWriteShredAction, budget::_LiveByteBudget) + plan = action.plan::_NestedWriteStructPlan + shape = plan.shape + index = action.position + if shape.package_owned + value = action.value::StructValue + child = _nestedwritestructaccesschild(value, value.names, + value.children, action.count, index, shape.names[index], + something(shape.source_children)[index]) + snapshot = _nestedwriteoccurrencesnapshot!(context.trace, child, + plan.children[index].shape, context.limits) + item, child_witness = _nestedwriterowaccess(child, value.index, + snapshot) + if index < action.count + _nestedwritestackpush!(stack, _NestedWriteShredAction(action.kind, + plan, value, action.expected, action.keymode, + action.repetition, action.row_witness, nothing, nothing, + nothing, nothing, nothing, nothing, index + 1, action.count, + 0, 0, UInt8(0)), budget) + end + _nestedwritestackpush!(stack, _nestedwriteshredpost(plan, + child_witness, action.row_witness), budget) + _nestedwritestackpush!(stack, _nestedwriteshredenter( + plan.children[index], item, + _nestedwriteshredchildexpected(action, index), action.keymode, + action.repetition, child_witness), budget) + else + item = getfield(action.value, index) + if index < action.count + _nestedwritestackpush!(stack, _NestedWriteShredAction(action.kind, + plan, action.value, action.expected, action.keymode, + action.repetition, action.row_witness, nothing, nothing, + nothing, nothing, nothing, nothing, index + 1, action.count, + 0, 0, UInt8(0)), budget) + end + _nestedwritestackpush!(stack, _nestedwriteshredenter( + plan.children[index], item, + _nestedwriteshredchildexpected(action, index), action.keymode, + action.repetition, nothing), budget) + end + return +end + +function _nestedwriteshredprocesslist!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, context, + action::_NestedWriteShredAction, budget::_LiveByteBudget) + plan = action.plan::_NestedWriteListPlan + index = action.position + _nestedwriterowcheck(action.row_witness) + _nestedwriteviewaccesscheck(action.value, action.count, action.first, + action.last) + ordinal = index - action.first + 1 + expected = _nestedwriteshredchildexpected(action, ordinal) + repeated = UInt64(plan.semantic.repetition_level) + itemrepetition = ordinal == 1 ? action.repetition : repeated + item, child_witness = _nestedwritelistrowaccess(action.value, index, + action.snapshot1) + _nestedwriteviewaccesscheck(action.value, action.count, action.first, + action.last) + index < action.last && _nestedwritestackpush!(stack, + _NestedWriteShredAction(action.kind, plan, action.value, + action.expected, action.keymode, action.repetition, + action.row_witness, nothing, nothing, nothing, nothing, + action.snapshot1, nothing, index + 1, action.count, action.first, + action.last, UInt8(0)), budget) + _nestedwritestackpush!(stack, _nestedwriteshredpost(plan, child_witness, + action.row_witness), budget) + _nestedwritestackpush!(stack, _nestedwriteshredenter(plan.element, item, + expected, action.keymode, itemrepetition, child_witness), budget) + return +end + +function _nestedwriteshredprocessdict!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, context, + action::_NestedWriteShredAction, budget::_LiveByteBudget) + entry = action.state + if action.phase == UInt8(1) + entry = entry.next + entry === nothing && return + end + plan = action.plan::_NestedWriteMapPlan + shape = plan.shape + index = action.position + ismissing(entry.key) && throw(ArgumentError( + "Parquet MAP field $(repr(shape.name)) contains a missing key")) + nested = _nestedwritetracekey!(context.trace, entry.key, shape.key, + context.limits) + itemrepetition = _nestedwriteshredmaprepetition(action, index) + _nestedwritestackpush!(stack, _NestedWriteShredAction(action.kind, plan, + action.value, action.expected, action.keymode, action.repetition, + action.row_witness, nothing, entry, nothing, action.materialization, + nothing, nothing, index + 1, action.count, 0, 0, UInt8(1)), budget) + mapvalue = shape.source_has_values ? entry.value : missing + _nestedwritestackpush!(stack, _nestedwriteshredenter(plan.value, mapvalue, + _nestedwriteshredmapvalueexpected(action, index), action.keymode, + itemrepetition, nothing), budget) + _nestedwritestackpush!(stack, _NestedWriteShredAction( + _NESTED_WRITE_SHRED_KEYASSERT, plan.key, entry.key, nothing, false, + UInt64(0), nothing, nothing, nothing, nested, nothing, nothing, nothing, + 0, 0, 0, 0, UInt8(0)), budget) + keyexpected = _nestedwriteshredmapkeyexpected(action, index, nested) + _nestedwritestackpush!(stack, _nestedwriteshredenter(plan.key, entry.key, + keyexpected, _nestedwriteshredkeymode(action, nested), itemrepetition, + nothing), budget) + return +end + +function _nestedwriteshredprocessviewkey!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, context, + action::_NestedWriteShredAction, budget::_LiveByteBudget) + plan = action.plan::_NestedWriteMapPlan + shape = plan.shape + entry = action.position + _nestedwriterowcheck(action.row_witness) + _nestedwriteviewaccesscheck(action.value, action.count, action.first, + action.last) + physical = action.value.first + entry - 1 + keyvalue, key_witness = _nestedwriterowaccess(action.value.keys, physical, + action.snapshot1) + if !action.keymode + _nestedwriterowcheck(action.row_witness) + if shape.source_has_values + rawvalue, value_witness = _nestedwriterowaccess( + something(action.value.values), physical, action.snapshot2) + else + rawvalue = missing + value_witness = nothing + end + else + rawvalue = nothing + value_witness = nothing + end + _nestedwriterowcheck(key_witness) + ismissing(keyvalue) && throw(ArgumentError( + "Parquet MAP field $(repr(shape.name)) contains a missing key")) + nested = _nestedwritetracekey!(context.trace, keyvalue, shape.key, + context.limits, nothing, key_witness) + itemrepetition = _nestedwriteshredmaprepetition(action, entry) + _nestedwritestackpush!(stack, _NestedWriteShredAction( + _NESTED_WRITE_SHRED_MAP_VIEW_VALUE, plan, action.value, + action.expected, action.keymode, action.repetition, action.row_witness, + key_witness, keyvalue, nested, nothing, action.snapshot1, + action.snapshot2, physical, action.count, action.first, action.last, + rawvalue, value_witness, UInt8(0)), budget) + keyexpected = _nestedwriteshredmapkeyexpected(action, entry, nested) + _nestedwritestackpush!(stack, _nestedwriteshredenter(plan.key, keyvalue, + keyexpected, _nestedwriteshredkeymode(action, nested), itemrepetition, + key_witness), budget) + return +end + +function _nestedwriteshredprocessviewvalue!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, context, + action::_NestedWriteShredAction, budget::_LiveByteBudget) + plan = action.plan::_NestedWriteMapPlan + shape = plan.shape + keyvalue = action.state + _nestedwritekeyassert!(action.nested, keyvalue, shape.key, + context.limits, context.trace, nothing, action.other_witness) + _nestedwriterowcheck(action.other_witness) + entry = action.position - action.value.first + 1 + itemrepetition = _nestedwriteshredmaprepetition(action, entry) + if action.keymode + _nestedwriterowcheck(action.row_witness) + if shape.source_has_values + mapvalue, value_witness = _nestedwriterowaccess( + something(action.value.values), action.position, + action.snapshot2) + else + mapvalue = missing + value_witness = nothing + end + else + mapvalue = action.rawvalue + value_witness = action.value_witness + end + entry < action.count && _nestedwritestackpush!(stack, + _NestedWriteShredAction(_NESTED_WRITE_SHRED_MAP_VIEW_KEY, plan, + action.value, action.expected, action.keymode, action.repetition, + action.row_witness, nothing, nothing, nothing, nothing, + action.snapshot1, action.snapshot2, entry + 1, + action.count, action.first, action.last, UInt8(0)), budget) + _nestedwritestackpush!(stack, _nestedwriteshredpost(plan, value_witness, + action.row_witness), budget) + _nestedwritestackpush!(stack, _nestedwriteshredenter(plan.value, mapvalue, + _nestedwriteshredmapvalueexpected(action, entry), action.keymode, + itemrepetition, value_witness), budget) + return +end + +function _nestedwriteshredprocess!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, + context, action::_NestedWriteShredAction, budget::_LiveByteBudget) + kind = action.kind + plan = action.plan + if kind == _NESTED_WRITE_SHRED_ENTER + if plan isa _NestedWriteLeafPlan + if action.keymode + return _nestedwriteshredkey!(context, plan, action.value, + something(action.expected), action.repetition, + action.row_witness) + end + return _nestedwriteshred!(context, plan, action.value, + action.repetition, action.row_witness) + elseif plan isa _NestedWriteStructPlan + return _nestedwriteshredstructenter!(stack, context, plan, + action.value, action.expected, action.keymode, + action.repetition, action.row_witness, budget) + elseif plan isa _NestedWriteListPlan + return _nestedwriteshredlistenter!(stack, context, plan, + action.value, action.expected, action.keymode, + action.repetition, action.row_witness, budget) + end + return _nestedwriteshredmapenter!(stack, context, + plan::_NestedWriteMapPlan, action.value, action.expected, + action.keymode, action.repetition, action.row_witness, budget) + elseif kind == _NESTED_WRITE_SHRED_POSTCHECK + _nestedwriterowcheck(action.row_witness) + _nestedwriterowcheck(action.other_witness) + return + elseif kind == _NESTED_WRITE_SHRED_KEYASSERT + _nestedwritekeyassert!(action.nested, action.value, plan.shape, + context.limits, context.trace, nothing, action.row_witness) + return + elseif kind == _NESTED_WRITE_SHRED_DICT_RELEASE + _nestedwritedictrelease!(context.trace, + something(action.materialization)) + return + elseif kind == _NESTED_WRITE_SHRED_STRUCT + return _nestedwriteshredprocessstruct!(stack, context, action, budget) + elseif kind == _NESTED_WRITE_SHRED_LIST + return _nestedwriteshredprocesslist!(stack, context, action, budget) + elseif kind == _NESTED_WRITE_SHRED_MAP_DICT + return _nestedwriteshredprocessdict!(stack, context, action, budget) + elseif kind == _NESTED_WRITE_SHRED_MAP_VIEW_KEY + return _nestedwriteshredprocessviewkey!(stack, context, action, budget) + elseif kind == _NESTED_WRITE_SHRED_MAP_VIEW_VALUE + return _nestedwriteshredprocessviewvalue!(stack, context, action, + budget) + end + return +end + +function _nestedwriteshredcleanup!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, + context) + for action in Iterators.reverse(stack.frames) + action.kind == _NESTED_WRITE_SHRED_DICT_RELEASE || continue + _nestedwritedictrelease!(context.trace, + something(action.materialization)) + end + return +end + +function _nestedwriteshrediterative!( + stack::_NestedWritePassStack{_NestedWriteShredAction}, + context, plan::_NestedWriteNodePlan, value, + expected::Union{Nothing,_NestedWriteKeySnapshot}, keymode::Bool, + repetition::UInt64, row_witness, budget::_LiveByteBudget) + Base.@nospecialize value + isempty(stack.frames) && !stack.processing || throw(AssertionError( + "nested writer shred scratch stack is not empty")) + try + _nestedwritestackpush!(stack, _nestedwriteshredenter(plan, value, + expected, keymode, repetition, row_witness), budget) + while !isempty(stack.frames) + action = _nestedwritepassstackpop!(stack) + try + _nestedwriteshredprocess!(stack, context, action, budget) + finally + _nestedwritepassstackprocessed!(stack) + end + end + finally + if !isempty(stack.frames) + try + _nestedwriteshredcleanup!(stack, context) + finally + _nestedwritepassstackclear!(stack) + end + end + end + isempty(stack.frames) && !stack.processing || throw(AssertionError( + "nested writer shred scratch stack retained actions")) + return +end + +function _nestedwriteshrediterative!(context, plan::_NestedWriteNodePlan, + value, expected::Union{Nothing,_NestedWriteKeySnapshot}, keymode::Bool, + repetition::UInt64, row_witness) + Base.@nospecialize value + trace = context.trace + budget = trace isa _NestedWriteTrace ? trace.budget : + _LiveByteBudget(context.limits) + stack = _nestedwritepassstackstart(_NestedWriteShredAction, budget) + try + _nestedwriteshrediterative!(stack, context, plan, value, expected, + keymode, repetition, row_witness, budget) + finally + _nestedwritepassstackrelease!(stack, budget) + end + return +end + +function _nestedwriteshred!(context, plan::_NestedWriteContainerPlan, value, + repetition::UInt64, row_witness) + return _nestedwriteshrediterative!(context, plan, value, nothing, false, + repetition, row_witness) +end + +function _nestedwriteshred!(context, plan::_NestedWriteContainerPlan, value, + repetition::UInt64) + return _nestedwriteshred!(context, plan, value, repetition, nothing) +end + +function _nestedwriteshredkey!(context, plan::_NestedWriteContainerPlan, value, + expected::_NestedWriteKeySnapshot, repetition::UInt64, row_witness) + return _nestedwriteshrediterative!(context, plan, value, expected, true, + repetition, row_witness) +end + +function _nestedwriteshredkey!(context, plan::_NestedWriteContainerPlan, value, + expected::_NestedWriteKeySnapshot, repetition::UInt64) + return _nestedwriteshredkey!(context, plan, value, expected, repetition, + nothing) +end + +function _nestedwritebuilderbytes(count::_NestedWriteLeafCount, + physical::Type) + bytes = _materializedarraybytes(UInt64, count.entries) + bytes = _materializedsum(bytes, + _materializedarraybytes(UInt64, count.entries)) + bytes = _materializedsum(bytes, + _materializedarraybytes(physical, count.dense)) + if physical === Vector{UInt8} + bytes = _materializedsum(bytes, _materializedproduct(count.dense, + _MATERIALIZED_ARRAY_HEADER_BYTES)) + bytes = _materializedsum(bytes, count.payload_bytes) + end + return _materializedsum(bytes, _MATERIALIZED_OBJECT_BYTES) +end + +function _nestedwritebuilders(counts::Vector{_NestedWriteLeafCount}, + semantic::_NestedSchemaPlan, budget::_LiveByteBudget) + length(counts) == length(semantic.leaves) || throw(AssertionError( + "nested writer count and leaf totals differ")) + charge = _materializedarraybytes(_NestedWriteLeafBuilder, length(counts)) + for (count, leaf) in zip(counts, semantic.leaves) + physical = _physicaleltype(leaf.source.element.type_) + charge = _materializedsum(charge, + _nestedwritebuilderbytes(count, physical)) + end + _reserve!(budget, charge) + builders = _NestedWriteLeafBuilder[] + sizehint!(builders, length(counts)) + for (count, leaf) in zip(counts, semantic.leaves) + physical = _physicaleltype(leaf.source.element.type_) + entries = Int(count.entries) + dense = Int(count.dense) + push!(builders, _NestedWriteLeafBuilder( + Vector{UInt64}(undef, entries), Vector{UInt64}(undef, entries), + Vector{physical}(undef, dense), 0, 0, Int64(0))) + end + return builders +end + +function _nestedwritevalidatebuilders(builders::Vector{_NestedWriteLeafBuilder}, + counts::Vector{_NestedWriteLeafCount}, semantic::_NestedSchemaPlan, + rows::Int) + for index in eachindex(builders, counts, semantic.leaves) + builder = builders[index] + count = counts[index] + leaf = semantic.leaves[index] + builder.entry_position == count.entries || throw(ArgumentError( + "nested Parquet input changed its level-entry count between passes")) + builder.dense_position == count.dense || throw(ArgumentError( + "nested Parquet input changed its dense-value count between passes")) + builder.payload_position == count.payload_bytes || throw(ArgumentError( + "nested Parquet input changed its variable-width payload between passes")) + LeafStream(builder.repetition, builder.definition, builder.values, + leaf.source.max_repetition_level, leaf.source.max_definition_level; + expected_rows=rows) + end + return +end + + +function _nestedwritecolumn(builder::_NestedWriteLeafBuilder, + leaf::_NestedLeafPlan, rows::Int, budget::_LiveByteBudget) + node = leaf.source + element = node.element + pathcharge = _reservearray!(budget, String, length(node.path)) + pathcharge >= 0 || throw(AssertionError( + "nested writer path charge is negative")) + _reserveobjects!(budget) + optional = element.repetition_type == Metadata.FieldRepetitionType.OPTIONAL + return WriteColumn(element.name, builder.values, element.type_, + element.type_length, optional, element.logicalType, + element.converted_type, copy(node.path), builder.repetition, + builder.definition, node.max_repetition_level, + node.max_definition_level, rows, Metadata.SchemaElement[]) +end + +function _nestedwritefinishfields(fragments::Vector{Vector{Metadata.SchemaElement}}, + plans::Vector{_NestedWriteNodePlan}, semantic::_NestedSchemaPlan, + builders::Vector{_NestedWriteLeafBuilder}, rows::Int, + budget::_LiveByteBudget) + length(fragments) == length(plans) || throw(AssertionError( + "nested writer field schema and plan counts differ")) + _reservearray!(budget, WriteFieldPlan, length(plans)) + fields = WriteFieldPlan[] + sizehint!(fields, length(plans)) + for (fragment, plan) in zip(fragments, plans) + range = getfield(getfield(plan, :semantic), :leaf_range) + isempty(range) && throw(ArgumentError( + "zero-leaf fields cannot be written to Parquet")) + _reservearray!(budget, WriteColumn, length(range)) + leaves = WriteColumn[] + sizehint!(leaves, length(range)) + for rawindex in range + index = Int(rawindex) + push!(leaves, _nestedwritecolumn(builders[index], + semantic.leaves[index], rows, budget)) + end + _reserveobjects!(budget) + push!(fields, WriteFieldPlan(fragment, leaves)) + end + return fields +end + +function _nestedwritecompile(shapes::Vector{_NestedWriteShape}, + fragments::Vector{Vector{Metadata.SchemaElement}}, limits::Limits, + budget::_LiveByteBudget) + length(shapes) == length(fragments) || throw(AssertionError( + "nested writer shape and schema fragment counts differ")) + length(shapes) <= typemax(Int32) || throw(ArgumentError( + "Parquet schema has more than Int32 top-level fields")) + total = 1 + for fragment in fragments + total = try + Base.checked_add(total, length(fragment)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, total, + limits.max_container_elements) + end + _reservearray!(budget, Metadata.SchemaElement, total) + elements = Metadata.SchemaElement[Metadata.SchemaElement( + name="schema", num_children=Int32(length(shapes)))] + sizehint!(elements, total) + for fragment in fragments + append!(elements, fragment) + end + schema = Schema(elements; limits=limits, budget=budget) + semantic = _nestedplan(schema; limits=limits, budget=budget) + length(semantic.root.children) == length(shapes) || throw(AssertionError( + "nested writer top-level schema changed during semantic compilation")) + _reservearray!(budget, _NestedWriteNodePlan, length(shapes)) + plans = _NestedWriteNodePlan[] + sizehint!(plans, length(shapes)) + for (shape, child) in zip(shapes, semantic.root.children) + push!(plans, _nestedwritebind(shape, child, budget)) + end + return semantic, plans +end + +function _nestedwritecounts(leaves::Int, budget::_LiveByteBudget) + _reservearray!(budget, _NestedWriteLeafCount, leaves) + _reserveobjects!(budget, leaves) + counts = _NestedWriteLeafCount[] + sizehint!(counts, leaves) + for _ in 1:leaves + push!(counts, _NestedWriteLeafCount(Int64(0), Int64(0), Int64(0))) + end + return counts +end + +function _nestedwriteboundarybytes(leaves::Int, rows::Int) + bytes = _materializedproduct(3, + _materializedarraybytes(Vector{Int64}, leaves)) + inner = _materializedarraybytes(Int64, rows + 1) + bytes = _materializedsum(bytes, + _materializedproduct(3 * leaves, inner)) + return _materializedsum(bytes, _MATERIALIZED_OBJECT_BYTES) +end + +function _nestedwritecountcontext(counts::Vector{_NestedWriteLeafCount}, + semantic::_NestedSchemaPlan, rows::Int, limits::Limits, + budget::_LiveByteBudget, trace::Union{Nothing,_NestedWriteTrace}=nothing) + leaves = length(semantic.leaves) + length(counts) == leaves || throw(AssertionError( + "nested writer count and semantic leaf totals differ")) + charge = _nestedwriteboundarybytes(leaves, rows) + _reserve!(budget, charge) + entries = Vector{Vector{Int64}}(undef, leaves) + dense = Vector{Vector{Int64}}(undef, leaves) + payload = Vector{Vector{Int64}}(undef, leaves) + for index in 1:leaves + entries[index] = zeros(Int64, rows + 1) + dense[index] = zeros(Int64, rows + 1) + payload[index] = zeros(Int64, rows + 1) + end + return _NestedWriteCountContext(counts, limits, semantic, entries, dense, + payload, trace), charge +end + +function _nestedwriterawcumulative(element::Metadata.SchemaElement, + dense::Int64, payload::Int64) + physical = element.type_ + physical == Metadata.Type.BOOLEAN && return cld(dense, Int64(8)) + physical in (Metadata.Type.INT32, Metadata.Type.FLOAT) && + return Base.checked_mul(dense, Int64(4)) + physical in (Metadata.Type.INT64, Metadata.Type.DOUBLE) && + return Base.checked_mul(dense, Int64(8)) + physical == Metadata.Type.BYTE_ARRAY && return Base.checked_add(payload, + Base.checked_mul(dense, Int64(4))) + physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY && return payload + throw(ArgumentError("unsupported writer physical type $physical")) +end + +function _nestedwritefinishrow!(context::_NestedWriteCountContext, + range, row::Int) + context.entry_offsets === nothing && return + semantic = something(context.semantic) + for rawindex in range + index = Int(rawindex) + count = context.counts[index] + context.entry_offsets[index][row + 1] = count.entries + context.dense_offsets[index][row + 1] = count.dense + element = semantic.leaves[index].source.element + context.payload_offsets[index][row + 1] = + _nestedwriterawcumulative(element, count.dense, + count.payload_bytes) + end + return +end + +function _nestedwritefinishrow!(context::_NestedWriteEmitContext, + range, row::Int) + context.boundaries === nothing && return + boundaries = context.boundaries + semantic = something(boundaries.semantic) + for rawindex in range + index = Int(rawindex) + builder = context.builders[index] + builder.entry_position == boundaries.entry_offsets[index][row + 1] || + throw(ArgumentError( + "nested Parquet input changed its per-row level-entry count between passes")) + builder.dense_position == boundaries.dense_offsets[index][row + 1] || + throw(ArgumentError( + "nested Parquet input changed its per-row dense-value count between passes")) + element = semantic.leaves[index].source.element + payload = _nestedwriterawcumulative(element, + Int64(builder.dense_position), builder.payload_position) + payload == boundaries.payload_offsets[index][row + 1] || + throw(ArgumentError( + "nested Parquet input changed its per-row physical payload between passes")) + end + return +end + +function _nestedwritevalidateboundaries(context::_NestedWriteCountContext, + rows::Int) + context.entry_offsets === nothing && return + semantic = something(context.semantic) + for index in eachindex(context.counts, semantic.leaves) + entries = context.entry_offsets[index] + dense = context.dense_offsets[index] + payload = context.payload_offsets[index] + length(entries) == rows + 1 == length(dense) == length(payload) || + throw(AssertionError("nested writer count-prefix lengths differ")) + first(entries) == first(dense) == first(payload) == 0 || + throw(AssertionError("nested writer count prefixes do not start at zero")) + for row in 1:rows + entries[row + 1] > entries[row] || throw(ArgumentError( + "every top-level row must add a level entry to every leaf")) + dense[row + 1] >= dense[row] && payload[row + 1] >= payload[row] || + throw(AssertionError("nested writer count prefixes are not monotonic")) + end + count = context.counts[index] + last(entries) == count.entries && last(dense) == count.dense || + throw(AssertionError("nested writer count prefixes have wrong terminals")) + element = semantic.leaves[index].source.element + last(payload) == _nestedwriterawcumulative(element, count.dense, + count.payload_bytes) || throw(AssertionError( + "nested writer payload prefix has the wrong terminal")) + end + return +end + +function _nestedwriterowlowerbound(element::Metadata.SchemaElement, + choice::Union{Nothing,WriteEncodingChoice}, raw::Int64) + choice === nothing && return raw + choice.dictionary && return Int64(0) + encoding = choice.encoding + encoding === nothing && return raw + encoding in (Metadata.Encoding.PLAIN, + Metadata.Encoding.BYTE_STREAM_SPLIT) && return raw + return Int64(0) +end + +function _nestedwritepreflightrows(context::_NestedWriteCountContext, + choices::Union{Nothing,Vector{WriteEncodingChoice}}) + context.entry_offsets === nothing && return + semantic = something(context.semantic) + choices === nothing || length(choices) == length(semantic.leaves) || + throw(AssertionError("writer preflight choice and leaf counts differ")) + rows = length(first(context.entry_offsets)) - 1 + for index in eachindex(semantic.leaves) + entries = context.entry_offsets[index] + dense = context.dense_offsets[index] + payload = context.payload_offsets[index] + element = semantic.leaves[index].source.element + choice = choices === nothing ? nothing : choices[index] + for row in 1:rows + entrycount = entries[row + 1] - entries[row] + entrycount <= typemax(Int32) || throw(LimitError(:page_values, + entrycount, Int64(typemax(Int32)))) + raw = if element.type_ == Metadata.Type.BOOLEAN + cld(dense[row + 1] - dense[row], Int64(8)) + else + payload[row + 1] - payload[row] + end + lower = _nestedwriterowlowerbound(element, choice, raw) + lower <= context.limits.max_page_bytes || throw(LimitError( + :page_bytes, lower, context.limits.max_page_bytes)) + end + end + return +end + +function _nestedwritepackageaccesscheck(::AbstractVector) + return +end + +function _nestedwritepackageaccesscheck(values::ListVector) + _validatelistvector(values) + return +end + +function _nestedwritepackageaccesscheck(values::StructVector) + _validatestructvector(values) + return +end + +function _nestedwritepackageaccesscheck(values::MapVector) + _validatemapvector(values) + return +end + +function _nestedwritecolumnaccesscheck(values::AbstractVector, count::Int, + first::Int, last::Int) + _nestedvectorcount(values, "nested Parquet input") == count || throw(ArgumentError( + "nested Parquet input changed its vector length during a writer pass")) + currentfirst, currentlast = _nestedvectoraxes(values, + "nested Parquet input") + currentfirst == first && currentlast == last || throw( + ArgumentError( + "nested Parquet input changed its vector axes during a writer pass")) + return +end + +function _nestedwritescanrows!(context, plans::Vector{_NestedWriteNodePlan}, + values::Vector{AbstractVector}, rows::Int) + length(plans) == length(values) || throw(AssertionError( + "nested writer plan and input column counts differ")) + trace = context.trace + budget = trace === nothing ? nothing : trace.budget + stack::Union{Nothing,_NestedWritePassStack{_NestedWriteShredAction}} = + nothing + try + for index in eachindex(plans, values) + plan = plans[index] + column = values[index] + expected = _nestedvectorcount(column, "nested Parquet column") + first, last = _nestedvectoraxes(column, "nested Parquet column") + shape = getfield(plan, :shape) + snapshot = _nestedwritesourcesnapshot(shape) + count = 0 + for rowindex in eachindex(column) + _nestedwritecolumnaccesscheck(column, expected, first, last) + count += 1 + value, row_witness = _nestedwriterowaccess(column, rowindex, + snapshot) + if plan isa _NestedWriteLeafPlan + _nestedwriteshred!(context, plan, value, UInt64(0), + row_witness) + else + if stack === nothing + budget === nothing && + (budget = _LiveByteBudget(context.limits)) + stack = _nestedwritepassstackstart( + _NestedWriteShredAction, something(budget)) + end + _nestedwriteshrediterative!(something(stack), context, + plan, value, nothing, false, UInt64(0), row_witness, + something(budget)) + end + _nestedwriterowcheck(row_witness) + range = getfield(getfield(plan, :semantic), :leaf_range) + _nestedwritefinishrow!(context, range, count) + end + count == rows || throw(ArgumentError( + "nested Parquet input changed its row count between passes")) + end + finally + stack === nothing || _nestedwritepassstackrelease!(stack, + something(budget)) + end + return +end + +function _nestedwritevalidatedinputs(input_columns::AbstractVector, rows::Int, + limits::Limits, budget::_LiveByteBudget) + isempty(input_columns) && throw(ArgumentError( + "a Parquet table must have at least one column")) + _checklimit(:container_elements, length(input_columns), + limits.max_container_elements) + _reservearray!(budget, String, length(input_columns)) + _reservearray!(budget, AbstractVector, length(input_columns)) + _reserveobjects!(budget) + names = String[] + values = AbstractVector[] + seen = Set{String}() + sizehint!(names, length(input_columns)) + sizehint!(values, length(input_columns)) + for column in input_columns + name = _nestedwriteinputname(column) + occursin('\0', name) && throw(ArgumentError( + "Parquet top-level field names cannot contain NUL")) + name in seen && throw(ArgumentError( + "Parquet column names must be unique")) + _reserveobjects!(budget) + push!(seen, name) + columnvalues = _nestedwriteinputvalues(column) + columnvalues isa AbstractVector || throw(ArgumentError( + "Parquet columns must be vectors")) + _nestedwritecheckcontainer(columnvalues, limits, + "Parquet column $(repr(name))") + count = _nestedvectorcount(columnvalues, + "Parquet column $(repr(name))") + count == rows || throw(ArgumentError( + "Parquet column $(repr(name)) has $count rows; " * + "expected $rows")) + push!(names, name) + push!(values, columnvalues) + end + return names, values +end + +function _nestedwriteshapes(names::Vector{String}, + values::Vector{AbstractVector}, limits::Limits, + budget::_LiveByteBudget, + topology::_NestedWriteTopologySnapshot) + _reservearray!(budget, _NestedWriteShape, length(values)) + shapes = _NestedWriteShape[] + sizehint!(shapes, length(values)) + for index in eachindex(names, values) + push!(shapes, _nestedwriteshape(names[index], eltype(values[index]), + values[index], limits, budget; topology=topology)) + end + return shapes +end + +function _nestedwritescanaggregates!(shapes::Vector{_NestedWriteShape}, + values::Vector{AbstractVector}, rows::Int, limits::Limits, + trace::Union{Nothing,_NestedWriteTrace}=nothing) + budget = trace === nothing ? nothing : trace.budget + stack::Union{Nothing,_NestedWritePassStack{_NestedWriteScanAction}} = + nothing + try + for index in eachindex(shapes, values) + shape = shapes[index] + column = values[index] + expected = _nestedvectorcount(column, "nested Parquet column") + first, last = _nestedvectoraxes(column, "nested Parquet column") + snapshot = _nestedwritesourcesnapshot(shape) + count = 0 + for rowindex in eachindex(column) + _nestedwritecolumnaccesscheck(column, expected, first, last) + count += 1 + value, row_witness = _nestedwriterowaccess(column, rowindex, + snapshot) + if shape isa _NestedWriteLeafShape + _nestedwritescanleaf!(shape, value, trace, row_witness) + else + if stack === nothing + budget === nothing && + (budget = _LiveByteBudget(limits)) + stack = _nestedwritepassstackstart( + _NestedWriteScanAction, something(budget)) + end + _nestedwritescanaggregate!(something(stack), shape, value, + limits, trace, row_witness, something(budget)) + end + _nestedwriterowcheck(row_witness) + end + count == rows || throw(ArgumentError( + "nested Parquet input changed its row count during schema inference")) + end + finally + stack === nothing || _nestedwritepassstackrelease!(stack, + something(budget)) + end + return +end + +function _nestedwriteschemafragments(shapes::Vector{_NestedWriteShape}, + limits::Limits, budget::_LiveByteBudget) + _reservearray!(budget, Vector{Metadata.SchemaElement}, length(shapes)) + fragments = Vector{Vector{Metadata.SchemaElement}}(undef, length(shapes)) + for index in eachindex(shapes) + fragments[index] = _nestedwriteschema(shapes[index], limits, budget) + end + return fragments +end + +function _nestedwritefields(input_columns::AbstractVector, rows::Integer, + limits::Limits, budget::_LiveByteBudget; preflight=nothing, + sourcevalidator=nothing) + rows >= 0 || throw(ArgumentError("Parquet row count must be nonnegative")) + rows <= typemax(Int) || throw(ArgumentError( + "Parquet row count exceeds the Julia index range")) + rowcount = Int(rows) + _checklimit(:container_elements, rowcount, + limits.max_container_elements) + start = _budgetused(budget) + try + names, values = _nestedwritevalidatedinputs(input_columns, rowcount, + limits, budget) + topology = _nestedwritetopology(input_columns, names, values, limits, + budget) + _nestedwritebarrier!(topology, sourcevalidator, budget) + shapes = _nestedwriteshapes(names, values, limits, budget, topology) + _nestedwritebarrier!(topology, sourcevalidator, budget) + trace = _nestedwritetrace(budget, topology) + _nestedwritescanaggregates!(shapes, values, rowcount, limits, trace) + _nestedwritebarrier!(topology, sourcevalidator, budget) + fragments = _nestedwriteschemafragments(shapes, limits, budget) + semantic, plans = _nestedwritecompile(shapes, fragments, limits, budget) + _nestedwritebarrier!(topology, sourcevalidator, budget) + choices, choicecharge = preflight === nothing ? + (nothing, Int64(0)) : preflight(semantic) + _nestedwritebarrier!(topology, sourcevalidator, budget) + counts = _nestedwritecounts(length(semantic.leaves), budget) + countcontext, boundarycharge = _nestedwritecountcontext(counts, + semantic, rowcount, limits, budget, trace) + _nestedwritetracecompare!(trace) + _nestedwritescanrows!(countcontext, plans, values, rowcount) + _nestedwritetracefinishcompare!(trace) + _nestedwritebarrier!(topology, sourcevalidator, budget) + _nestedwritevalidateboundaries(countcontext, rowcount) + _nestedwritepreflightrows(countcontext, choices) + iszero(choicecharge) || _release!(budget, choicecharge) + choices = nothing + builders = _nestedwritebuilders(counts, semantic, budget) + _nestedwritebarrier!(topology, sourcevalidator, budget) + emitcontext = _NestedWriteEmitContext(builders, counts, limits, + countcontext, trace) + _nestedwritetracecompare!(trace) + _nestedwritescanrows!(emitcontext, plans, values, rowcount) + _nestedwritetracefinishcompare!(trace) + _nestedwritebarrier!(topology, sourcevalidator, budget) + _nestedwritevalidatebuilders(builders, counts, semantic, rowcount) + fields = _nestedwritefinishfields(fragments, plans, semantic, builders, + rowcount, budget) + _nestedwritebarrier!(topology, sourcevalidator, budget) + _release!(budget, boundarycharge) + countcontext = nothing + emitcontext = nothing + _nestedwritetracerelease!(trace) + trace = nothing + _nestedwritetopologyrelease!(topology, budget) + topology = nothing + return fields + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end diff --git a/src/write_provenance.jl b/src/write_provenance.jl new file mode 100644 index 0000000..d86079d --- /dev/null +++ b/src/write_provenance.jl @@ -0,0 +1,1919 @@ +# Exact schema-bearing writes for materialized Parquet.Table values. + +abstract type _ProvenanceBinding end + +struct _ProvenanceLeafBinding <: _ProvenanceBinding + plan::_NestedLeafPlan + values::AbstractVector + force_required::Bool + snapshot::_NestedWriteVectorSnapshot +end + +struct _ProvenanceStructBinding <: _ProvenanceBinding + plan::_NestedStructPlan + values::StructVector + children::Vector{_ProvenanceBinding} + force_required::Bool + snapshot::_NestedWriteVectorSnapshot +end + +struct _ProvenanceListBinding <: _ProvenanceBinding + plan::_NestedListPlan + values::ListVector + element::_ProvenanceBinding + force_required::Bool + snapshot::_NestedWriteVectorSnapshot +end + +struct _ProvenanceMapBinding <: _ProvenanceBinding + plan::_NestedMapPlan + values::MapVector + key::_ProvenanceBinding + value::Union{Nothing,_ProvenanceBinding} + force_required::Bool + snapshot::_NestedWriteVectorSnapshot +end + +mutable struct _ProvenanceKeyReplayFrame + parent::Union{Nothing,_ProvenanceKeyReplayFrame} + binding::_ProvenanceBinding + snapshot::_NestedWriteKeySnapshot + repetition::UInt64 + depth::Int + position::Int +end + +mutable struct _ProvenanceSchemaCompareFrame + parent::Union{Nothing,_ProvenanceSchemaCompareFrame} + fresh::SchemaNode + stored::SchemaNode + depth::Int + position::Int +end + +mutable struct _ProvenancePlanWalkFrame + parent::Union{Nothing,_ProvenancePlanWalkFrame} + plan::_NestedPlan + position::Int + expected::Int +end + +mutable struct _ProvenanceSchemaWalkFrame + parent::Union{Nothing,_ProvenanceSchemaWalkFrame} + node::SchemaNode + depth::Int + position::Int +end + +const _PROVENANCE_BIND_STRUCT = UInt8(1) +const _PROVENANCE_BIND_LIST = UInt8(2) +const _PROVENANCE_BIND_MAP = UInt8(3) + +mutable struct _ProvenanceBindFrame + parent::Union{Nothing,_ProvenanceBindFrame} + plan::_NestedPlan + values::AbstractVector + force_required::Bool + mode::UInt8 + children::Union{Nothing,Vector{_ProvenanceBinding}} + firstbinding::Union{Nothing,_ProvenanceBinding} + secondbinding::Union{Nothing,_ProvenanceBinding} + expected::Int + completed::Int +end + +mutable struct _ProvenanceValidateFrame + parent::Union{Nothing,_ProvenanceValidateFrame} + binding::_ProvenanceBinding + childcount::Int + position::Int + expected::Int +end + +mutable struct _ProvenanceShredFrame + parent::Union{Nothing,_ProvenanceShredFrame} + binding::_ProvenanceBinding + repetition::UInt64 + childindex::Int + position::Int + last::Int + witness::_NestedWriteRowWitness +end + +function _provenanceframesrelease!(budget::_LiveByteBudget, count::Int) + iszero(count) && return + _release!(budget, + _materializedproduct(count, _MATERIALIZED_OBJECT_BYTES)) + return +end + +struct _ProvenanceWriteFields <: AbstractVector{WriteFieldPlan} + fields::Vector{WriteFieldPlan} + elements::Vector{Metadata.SchemaElement} + schema::Schema +end + +function Base.IndexStyle(::Type{_ProvenanceWriteFields}) + return IndexLinear() +end + +function Base.size(fields::_ProvenanceWriteFields) + return size(fields.fields) +end + +function Base.getindex(fields::_ProvenanceWriteFields, index::Int) + return fields.fields[index] +end + +function _provenanceexact(left::Thrift.RawField, right::Thrift.RawField) + return left.id == right.id && left.type == right.type && + left.previd == right.previd && left.headerlength == right.headerlength && + left.bytes == right.bytes +end + +function _provenanceexact(left::Vector{Thrift.RawField}, + right::Vector{Thrift.RawField}) + length(left) == length(right) || return false + for index in eachindex(left, right) + _provenanceexact(left[index], right[index]) || return false + end + return true +end + +function _provenanceexact(left, right) + typeof(left) === typeof(right) || return false + T = typeof(left) + if isstructtype(T) && hasfield(T, :unknown_fields) + for index in 1:fieldcount(T) + _provenanceexact(getfield(left, index), getfield(right, index)) || + return false + end + return true + end + return isequal(left, right) +end + +function _provenanceclone(value::Thrift.RawField) + return Thrift.RawField(value.id, value.type, value.previd, + value.headerlength, copy(value.bytes)) +end + +function _provenanceclone(value::Vector{Thrift.RawField}) + return map(_provenanceclone, value) +end + +function _provenanceclone(value) + T = typeof(value) + if isstructtype(T) && hasfield(T, :unknown_fields) + fields = ntuple(index -> _provenanceclone(getfield(value, index)), + fieldcount(T)) + return T(fields...) + end + return value +end + +function _provenanceclonecharge(value::Thrift.RawField) + return _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _materializedarraybytes(UInt8, length(value.bytes))) +end + +function _provenanceclonecharge(value::Vector{Thrift.RawField}) + bytes = _materializedarraybytes(Thrift.RawField, length(value)) + for item in value + bytes = _materializedsum(bytes, _provenanceclonecharge(item)) + end + return bytes +end + +function _provenanceclonecharge(value) + T = typeof(value) + isstructtype(T) && hasfield(T, :unknown_fields) || return Int64(0) + bytes = _MATERIALIZED_OBJECT_BYTES + for index in 1:fieldcount(T) + bytes = _materializedsum(bytes, + _provenanceclonecharge(getfield(value, index))) + end + return bytes +end + +function _provenancecomparelabel(label::String, index::Int) + iszero(index) && return label + return "$label $index" +end + +function _provenancecompareschemadirect(fresh::SchemaNode, + stored::SchemaNode, label::String, labelindex::Int) + _provenanceexact(fresh.element, stored.element) || throw(ArgumentError( + "$(_provenancecomparelabel(label, labelindex)) has a SchemaElement " * + "that differs from table.metadata.schema")) + fresh.path == stored.path || throw(ArgumentError( + "$(_provenancecomparelabel(label, labelindex)) has a path that differs " * + "from table.metadata.schema")) + fresh.max_definition_level == stored.max_definition_level || + throw(ArgumentError( + "$(_provenancecomparelabel(label, labelindex)) has a changed definition level")) + fresh.max_repetition_level == stored.max_repetition_level || + throw(ArgumentError( + "$(_provenancecomparelabel(label, labelindex)) has a changed repetition level")) + fresh.column_index == stored.column_index || throw(ArgumentError( + "$(_provenancecomparelabel(label, labelindex)) has a changed physical leaf ordinal")) + length(fresh.children) == length(stored.children) || throw(ArgumentError( + "$(_provenancecomparelabel(label, labelindex)) has changed child topology")) + return +end + +function _provenancecompareframe(fresh::SchemaNode, stored::SchemaNode, + label::String, labelindex::Int, parent, depth::Int, limits::Limits, + budget::_LiveByteBudget) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + _checklimit(:container_elements, length(fresh.children), + limits.max_container_elements) + _provenancecompareschemadirect(fresh, stored, label, labelindex) + _reserveobjects!(budget) + return _ProvenanceSchemaCompareFrame(parent, fresh, stored, depth, 0) +end + +function _provenancecompareschemanode(fresh::SchemaNode, + stored::SchemaNode, label::String, labelindex::Int, limits::Limits, + budget::_LiveByteBudget) + start = _budgetused(budget) + current::Union{Nothing,_ProvenanceSchemaCompareFrame} = nothing + activeframes = 0 + try + current = _provenancecompareframe(fresh, stored, label, labelindex, + nothing, 1, limits, budget) + activeframes = 1 + while true + if current.position == length(current.fresh.children) + parent = current.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + activeframes -= 1 + if parent === nothing + current = nothing + return + end + current = parent::_ProvenanceSchemaCompareFrame + continue + end + current.position += 1 + index = current.position + child = current.fresh.children[index] + storedchild = current.stored.children[index] + childdepth = _nestedwritedepthadd(current.depth, 1, limits) + current = _provenancecompareframe(child, storedchild, + "stored schema node", 0, current, childdepth, limits, budget) + activeframes += 1 + end + finally + _provenanceframesrelease!(budget, activeframes) + used = _budgetused(budget) + used >= start || throw(AssertionError( + "schema comparison released caller-owned budget")) + used > start && _release!(budget, used - start) + end +end + +function _provenancecompareschema(fresh::Schema, stored::Schema, + limits::Limits, budget::_LiveByteBudget) + _provenancecompareschemanode(fresh.root, stored.root, + "stored schema root", 0, limits, budget) + length(fresh.leaves) == length(stored.leaves) || throw(ArgumentError( + "stored schema has changed physical leaf ordering")) + for index in eachindex(fresh.leaves, stored.leaves) + _provenancecompareschemanode(fresh.leaves[index], stored.leaves[index], + "stored schema leaf", index, limits, budget) + end + return +end + +function _provenancefreshschema(table::Table, limits::Limits, + budget::_LiveByteBudget) + source = table.metadata.schema + _checklimit(:container_elements, length(source), limits.max_container_elements) + charge = _materializedarraybytes(Metadata.SchemaElement, length(source)) + for element in source + charge = _materializedsum(charge, _provenanceclonecharge(element)) + end + _reserve!(budget, charge) + elements = Vector{Metadata.SchemaElement}(undef, length(source)) + for index in eachindex(source) + elements[index] = _provenanceclone(source[index]) + end + schema = Schema(elements; limits=limits, budget=budget) + _provenancecompareschema(schema, table.schema, limits, budget) + return elements, schema +end + +function _provenancerequired(plan::_NestedPlan, force_required::Bool) + return force_required || getfield(plan, :parent_definition) == + getfield(plan, :present_definition) +end + +function _provenancepathlabel(prefix::String, node::SchemaNode, + suffix::String) + return "$prefix $(repr(join(node.path, ".")))$suffix" +end + +function _provenancecheckvector(values::AbstractVector, count::Int, + prefix::String, node::SchemaNode, suffix::String, limits::Limits) + length(values) == count || throw(ArgumentError( + "$(_provenancepathlabel(prefix, node, suffix)) has " * + "$(length(values)) values; expected $count")) + axes(values, 1) == Base.OneTo(count) || throw(ArgumentError( + "$(_provenancepathlabel(prefix, node, suffix)) must use one-based " * + "contiguous axes")) + _checklimit(:container_elements, count, limits.max_container_elements) + return +end + +function _provenancecontaineradd(value::Int, increment::Int, limits::Limits) + requested = try + Base.checked_add(value, increment) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, requested, limits.max_container_elements) + return requested +end + +function _provenanceleaflogicaltype(plan::_NestedLeafPlan) + element = plan.source.element + physical = _physicaleltype(element.type_) + return _logicaleltype(element, physical) +end + +function _provenanceleafexpectedtype(plan::_NestedLeafPlan, + force_required::Bool) + logical = _provenanceleaflogicaltype(plan) + _provenancerequired(plan, force_required) && return logical + return Union{Missing,logical} +end + +function _provenancevalidateleaf(binding::_ProvenanceLeafBinding, count::Int, + limits::Limits) + plan = binding.plan + values = binding.values + _provenancecheckvector(values, count, "physical leaf", plan.source, "", + limits) + expected = _provenanceleafexpectedtype(plan, binding.force_required) + eltype(values) == expected || throw(ArgumentError( + "physical leaf $(repr(join(plan.source.path, "."))) has logical " * + "element type $(eltype(values)); expected $expected")) + element = plan.source.element + fixed = element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY && + _logicalkind(element) === nothing + if fixed + values isa FixedByteArrayVector || throw(ArgumentError( + "raw fixed leaf $(repr(join(plan.source.path, "."))) lost its width")) + values.width == element.type_length || throw(ArgumentError( + "raw fixed leaf $(repr(join(plan.source.path, "."))) changed width")) + else + values isa FixedByteArrayVector && throw(ArgumentError( + "logical leaf $(repr(join(plan.source.path, "."))) has an unexpected " * + "raw fixed-width wrapper")) + end + return +end + +function _provenancecheckindices(indices::AbstractVector{<:Integer}, count::Int, + prefix::String, node::SchemaNode, suffix::String, limits::Limits; + ranks::Bool=false) + expected = _provenancecontaineradd(count, 1, limits) + _provenancecheckvector(indices, expected, prefix, node, suffix, limits) + iszero(first(indices)) || throw(ArgumentError( + "$(_provenancepathlabel(prefix, node, suffix)) must start at zero")) + previous = Int64(0) + for index in 2:length(indices) + value = indices[index] + value >= previous || throw(ArgumentError( + "$(_provenancepathlabel(prefix, node, suffix)) must be nondecreasing")) + value <= typemax(Int) || throw(ArgumentError( + "$(_provenancepathlabel(prefix, node, suffix)) exceeds the Julia index range")) + ranks && value - previous > 1 && throw(ArgumentError( + "$(_provenancepathlabel(prefix, node, suffix)) has a rank difference above one")) + previous = Int64(value) + end + _checklimit(:container_elements, previous, limits.max_container_elements) + return Int(previous) +end + +function _provenancevalidatevalidity(validity::BitVector, + offsets::AbstractVector{<:Integer}, count::Int, prefix::String, + node::SchemaNode, limits::Limits) + _provenancecheckvector(validity, count, prefix, node, " validity", limits) + for index in 1:count + validity[index] || offsets[index] == offsets[index + 1] || + throw(ArgumentError("null $(_provenancepathlabel(prefix, node, "")) " * + "row $index has a nonempty span")) + end + return +end + +function _provenancevalidatestruct(binding::_ProvenanceStructBinding, + count::Int, limits::Limits) + plan = binding.plan + values = binding.values + length(values) == count && values.rows == count || throw(ArgumentError( + "struct $(repr(join(plan.source.path, "."))) changed its row count")) + length(values.names) == length(plan.children) || throw(ArgumentError( + "struct $(repr(join(plan.source.path, "."))) changed its field count")) + length(values.children) == length(plan.children) || throw(ArgumentError( + "struct $(repr(join(plan.source.path, "."))) changed its child count")) + for index in eachindex(plan.children) + values.names[index] == plan.children[index].source.element.name || + throw(ArgumentError("struct $(repr(join(plan.source.path, "."))) " * + "changed field name or order at position $index")) + end + required = _provenancerequired(plan, binding.force_required) + if required + values.ranks === nothing || throw(ArgumentError( + "required struct $(repr(join(plan.source.path, "."))) gained validity")) + childcount = count + else + values.ranks === nothing && throw(ArgumentError( + "optional struct $(repr(join(plan.source.path, "."))) lost validity")) + childcount = _provenancecheckindices(values.ranks, count, "struct", + plan.source, " ranks", limits; ranks=true) + end + return childcount +end + +function _provenancevalidatelist(binding::_ProvenanceListBinding, count::Int, + limits::Limits) + plan = binding.plan + values = binding.values + _provenancecheckvector(values, count, "list", plan.source, "", limits) + entries = _provenancecheckindices(values.offsets, count, "list", + plan.source, " offsets", limits) + required = _provenancerequired(plan, binding.force_required) + if required + values.validity === nothing || throw(ArgumentError( + "required list $(repr(join(plan.source.path, "."))) gained validity")) + else + values.validity === nothing && throw(ArgumentError( + "optional list $(repr(join(plan.source.path, "."))) lost validity")) + _provenancevalidatevalidity(values.validity, values.offsets, count, + "list", plan.source, limits) + end + length(values.values) == entries || throw(ArgumentError( + "list $(repr(join(plan.source.path, "."))) terminal offset changed")) + return entries +end + +function _provenancevalidatemap(binding::_ProvenanceMapBinding, count::Int, + limits::Limits) + plan = binding.plan + values = binding.values + _provenancecheckvector(values, count, "map", plan.source, "", limits) + entries = _provenancecheckindices(values.offsets, count, "map", + plan.source, " offsets", limits) + required = _provenancerequired(plan, binding.force_required) + if required + values.validity === nothing || throw(ArgumentError( + "required map $(repr(join(plan.source.path, "."))) gained validity")) + else + values.validity === nothing && throw(ArgumentError( + "optional map $(repr(join(plan.source.path, "."))) lost validity")) + _provenancevalidatevalidity(values.validity, values.offsets, count, + "map", plan.source, limits) + end + length(values.keys) == entries || throw(ArgumentError( + "map $(repr(join(plan.source.path, "."))) key count changed")) + if plan.value === nothing + values.values === nothing || throw(ArgumentError( + "key-only map $(repr(join(plan.source.path, "."))) gained values")) + else + values.values === nothing && throw(ArgumentError( + "map $(repr(join(plan.source.path, "."))) lost its values")) + length(values.values) == entries || throw(ArgumentError( + "map $(repr(join(plan.source.path, "."))) value count changed")) + end + return entries +end + +function _provenancevalidateframe(binding::_ProvenanceBinding, + childcount::Int, expected::Int, parent, budget::_LiveByteBudget) + _reserveobjects!(budget) + return _ProvenanceValidateFrame(parent, binding, childcount, 0, expected) +end + +function _provenancevalidatestart(binding::_ProvenanceLeafBinding, + count::Int, limits::Limits, ::Union{Nothing,_ProvenanceValidateFrame}, + ::_LiveByteBudget) + _provenancevalidateleaf(binding, count, limits) + return nothing +end + +function _provenancevalidatestart(binding::_ProvenanceStructBinding, + count::Int, limits::Limits, parent, budget::_LiveByteBudget) + childcount = _provenancevalidatestruct(binding, count, limits) + isempty(binding.children) && return nothing + return _provenancevalidateframe(binding, childcount, + length(binding.children), parent, budget) +end + +function _provenancevalidatestart(binding::_ProvenanceListBinding, + count::Int, limits::Limits, parent, budget::_LiveByteBudget) + childcount = _provenancevalidatelist(binding, count, limits) + return _provenancevalidateframe(binding, childcount, 1, parent, budget) +end + +function _provenancevalidatestart(binding::_ProvenanceMapBinding, + count::Int, limits::Limits, parent, budget::_LiveByteBudget) + childcount = _provenancevalidatemap(binding, count, limits) + expected = binding.value === nothing ? 1 : 2 + return _provenancevalidateframe(binding, childcount, expected, parent, + budget) +end + +function _provenancevalidatenext(frame::_ProvenanceValidateFrame) + frame.position += 1 + binding = frame.binding + if binding isa _ProvenanceStructBinding + child = binding.children[frame.position] + values = binding.values + values.children[frame.position] === _provenancesource(child) || + throw(ArgumentError("struct " * + "$(repr(join(binding.plan.source.path, "."))) changed child " * + "identity at position $(frame.position)")) + return child + elseif binding isa _ProvenanceListBinding + return binding.element + elseif binding isa _ProvenanceMapBinding + frame.position == 1 && return binding.key + return something(binding.value) + end + throw(AssertionError("unknown schema-bearing validation binding")) +end + +function _provenancevalidate(binding::_ProvenanceBinding, count::Int, + limits::Limits, budget::_LiveByteBudget) + start = _budgetused(budget) + current::Union{Nothing,_ProvenanceValidateFrame} = nothing + activeframes = 0 + try + current = _provenancevalidatestart(binding, count, limits, nothing, + budget) + current === nothing && return + activeframes = 1 + while true + if current.position == current.expected + parent = current.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + activeframes -= 1 + if parent === nothing + current = nothing + return + end + current = parent::_ProvenanceValidateFrame + continue + end + child = _provenancevalidatenext(current) + childframe = _provenancevalidatestart(child, current.childcount, + limits, current, budget) + if childframe !== nothing + activeframes += 1 + current = childframe::_ProvenanceValidateFrame + end + end + finally + _provenanceframesrelease!(budget, activeframes) + used = _budgetused(budget) + used >= start || throw(AssertionError( + "schema-bearing validation released caller-owned budget")) + used > start && _release!(budget, used - start) + end +end + +function _provenancebindstart(plan::_NestedLeafPlan, values, + force_required::Bool, ::Union{Nothing,_ProvenanceBindFrame}, + limits::Limits, budget::_LiveByteBudget, + topology::_NestedWriteTopologySnapshot) + depth = _nestedwritedepthadd(length(plan.source.path), 1, limits) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + values isa AbstractVector || throw(ArgumentError( + "physical leaf $(repr(join(plan.source.path, "."))) is not a vector")) + _reserveobjects!(budget) + return (_ProvenanceLeafBinding(plan, values, force_required, + _nestedwritesnapshotfor(topology, values)), nothing) +end + +function _provenancebindframe(parent, plan::_NestedPlan, + values::AbstractVector, force_required::Bool, mode::UInt8, + children::Union{Nothing,Vector{_ProvenanceBinding}}, expected::Int, + budget::_LiveByteBudget) + _reserveobjects!(budget) + return _ProvenanceBindFrame(parent, plan, values, force_required, mode, + children, nothing, nothing, expected, 0) +end + +function _provenancebindstart(plan::_NestedStructPlan, values, + force_required::Bool, parent, limits::Limits, + budget::_LiveByteBudget, ::_NestedWriteTopologySnapshot) + depth = _nestedwritedepthadd(length(plan.source.path), 1, limits) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + values isa StructVector || throw(ArgumentError( + "schema struct $(repr(join(plan.source.path, "."))) requires StructVector")) + length(values.children) == length(plan.children) || throw(ArgumentError( + "schema struct $(repr(join(plan.source.path, "."))) has a changed child count")) + _checklimit(:container_elements, length(plan.children), + limits.max_container_elements) + _reservearray!(budget, _ProvenanceBinding, length(plan.children)) + children = _ProvenanceBinding[] + sizehint!(children, length(plan.children)) + return (nothing, _provenancebindframe(parent, plan, values, + force_required, _PROVENANCE_BIND_STRUCT, children, + length(plan.children), budget)) +end + +function _provenancebindstart(plan::_NestedListPlan, values, + force_required::Bool, parent, limits::Limits, + budget::_LiveByteBudget, ::_NestedWriteTopologySnapshot) + depth = _nestedwritedepthadd(length(plan.source.path), 1, limits) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + values isa ListVector || throw(ArgumentError( + "schema list $(repr(join(plan.source.path, "."))) requires ListVector")) + return (nothing, _provenancebindframe(parent, plan, values, + force_required, _PROVENANCE_BIND_LIST, nothing, 1, budget)) +end + +function _provenancebindstart(plan::_NestedMapPlan, values, + force_required::Bool, parent, limits::Limits, + budget::_LiveByteBudget, ::_NestedWriteTopologySnapshot) + depth = _nestedwritedepthadd(length(plan.source.path), 1, limits) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + values isa MapVector || throw(ArgumentError( + "schema map $(repr(join(plan.source.path, "."))) requires MapVector")) + if plan.value === nothing + values.values === nothing || throw(ArgumentError( + "key-only schema map $(repr(join(plan.source.path, "."))) gained " * + "a value vector")) + expected = 1 + else + values.values === nothing && throw(ArgumentError( + "schema map $(repr(join(plan.source.path, "."))) lost its value vector")) + expected = 2 + end + return (nothing, _provenancebindframe(parent, plan, values, + force_required, _PROVENANCE_BIND_MAP, nothing, expected, budget)) +end + +function _provenancebindnext(frame::_ProvenanceBindFrame) + plan = frame.plan + values = frame.values + index = frame.completed + 1 + if frame.mode == _PROVENANCE_BIND_STRUCT + structplan = plan::_NestedStructPlan + structvalues = values::StructVector + return structplan.children[index], structvalues.children[index], false + elseif frame.mode == _PROVENANCE_BIND_LIST + return (plan::_NestedListPlan).element, + (values::ListVector).values, false + elseif frame.mode == _PROVENANCE_BIND_MAP + mapplan = plan::_NestedMapPlan + mapvalues = values::MapVector + index == 1 && return mapplan.key, mapvalues.keys, true + return something(mapplan.value), something(mapvalues.values), false + end + throw(AssertionError("unknown schema-bearing binding frame")) +end + +function _provenancebindaccept!(frame::_ProvenanceBindFrame, + binding::_ProvenanceBinding) + if frame.mode == _PROVENANCE_BIND_STRUCT + push!(frame.children::Vector{_ProvenanceBinding}, binding) + elseif frame.completed == 0 + frame.firstbinding = binding + else + frame.secondbinding = binding + end + frame.completed += 1 + return +end + +function _provenancebindfinish(frame::_ProvenanceBindFrame, + budget::_LiveByteBudget, topology::_NestedWriteTopologySnapshot) + _reserveobjects!(budget) + if frame.mode == _PROVENANCE_BIND_STRUCT + return _ProvenanceStructBinding(frame.plan::_NestedStructPlan, + frame.values::StructVector, + frame.children::Vector{_ProvenanceBinding}, frame.force_required, + _nestedwritesnapshotfor(topology, frame.values)) + elseif frame.mode == _PROVENANCE_BIND_LIST + return _ProvenanceListBinding(frame.plan::_NestedListPlan, + frame.values::ListVector, + frame.firstbinding::_ProvenanceBinding, frame.force_required, + _nestedwritesnapshotfor(topology, frame.values)) + end + return _ProvenanceMapBinding(frame.plan::_NestedMapPlan, + frame.values::MapVector, frame.firstbinding::_ProvenanceBinding, + frame.secondbinding, frame.force_required, + _nestedwritesnapshotfor(topology, frame.values)) +end + +function _provenancebind(plan::_NestedPlan, values, + limits::Limits, budget::_LiveByteBudget, + topology::_NestedWriteTopologySnapshot; + force_required::Bool=false) + pending, current = _provenancebindstart(plan, values, force_required, + nothing, limits, budget, topology) + pending === nothing || return pending::_ProvenanceBinding + active::Union{Nothing,_ProvenanceBindFrame} = + current::_ProvenanceBindFrame + activeframes = 1 + try + while true + if pending !== nothing + _provenancebindaccept!(active, pending) + pending = nothing + end + if active.completed == active.expected + pending = _provenancebindfinish(active, budget, topology) + parent = active.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + activeframes -= 1 + parent === nothing && begin + active = nothing + return pending::_ProvenanceBinding + end + active = parent::_ProvenanceBindFrame + continue + end + childplan, childvalues, required = _provenancebindnext(active) + pending, childframe = _provenancebindstart(childplan, childvalues, + required, active, limits, budget, topology) + if childframe !== nothing + activeframes += 1 + active = childframe::_ProvenanceBindFrame + end + end + finally + _provenanceframesrelease!(budget, activeframes) + end +end + +function _provenanceplanframe(plan::_NestedPlan, parent, + budget::_LiveByteBudget) + expected = plan isa _NestedStructPlan ? length(plan.children) : + plan isa _NestedListPlan ? 1 : plan isa _NestedMapPlan ? + (plan.value === nothing ? 1 : 2) : 0 + iszero(expected) && return nothing + _reserveobjects!(budget) + return _ProvenancePlanWalkFrame(parent, plan, 0, expected) +end + +function _provenanceplannext(frame::_ProvenancePlanWalkFrame) + frame.position += 1 + plan = frame.plan + plan isa _NestedStructPlan && return plan.children[frame.position] + plan isa _NestedListPlan && return plan.element + plan isa _NestedMapPlan && frame.position == 1 && return plan.key + plan isa _NestedMapPlan && return something(plan.value) + throw(AssertionError("unknown schema-bearing plan walk frame")) +end + +function _provenancerejectleafless(plan::_NestedPlan, + budget::_LiveByteBudget) + isempty(_nestedleafrange(plan)) && throw(ArgumentError( + "schema-bearing writes do not support zero-leaf group " * + repr(join(plan.source.path, ".")))) + current::Union{Nothing,_ProvenancePlanWalkFrame} = + _provenanceplanframe(plan, nothing, budget) + activeframes = current === nothing ? 0 : 1 + try + current === nothing && return + while true + if current.position == current.expected + parent = current.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + activeframes -= 1 + if parent === nothing + current = nothing + return + end + current = parent::_ProvenancePlanWalkFrame + continue + end + child = _provenanceplannext(current) + isempty(_nestedleafrange(child)) && throw(ArgumentError( + "schema-bearing writes do not support zero-leaf group " * + repr(join(child.source.path, ".")))) + childframe = _provenanceplanframe(child, current, budget) + if childframe !== nothing + activeframes += 1 + current = childframe::_ProvenancePlanWalkFrame + end + end + finally + _provenanceframesrelease!(budget, activeframes) + end +end + +function _provenancebindings(table::Table, semantic::_NestedSchemaPlan, + limits::Limits, budget::_LiveByteBudget, + topology::_NestedWriteTopologySnapshot) + start = _budgetused(budget) + try + columns = table.columns + values = Base.values(columns) + children = semantic.root.children + length(values) == length(children) == length(topology.names) || + throw(ArgumentError( + "table columns no longer match the stored Parquet schema")) + _reservearray!(budget, _ProvenanceBinding, length(children)) + bindings = _ProvenanceBinding[] + sizehint!(bindings, length(children)) + for index in eachindex(children) + topology.names[index] == children[index].source.element.name || + throw(ArgumentError("table column name or order no longer matches " * + "the stored Parquet schema at position $index")) + push!(bindings, _provenancebind(children[index], values[index], + limits, budget, topology)) + end + return bindings + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _provenancesource(binding::_ProvenanceLeafBinding) + return binding.values +end + +function _provenancesource(binding::_ProvenanceStructBinding) + return binding.values +end + +function _provenancesource(binding::_ProvenanceListBinding) + return binding.values +end + +function _provenancesource(binding::_ProvenanceMapBinding) + return binding.values +end + +function _nestedwritekeywitnesssnapshot(binding::_ProvenanceBinding) + return binding.snapshot +end + +function _nestedwritekeywitnessroot(binding::_ProvenanceLeafBinding, value) + if ismissing(value) + _provenancerequired(binding.plan, binding.force_required) && throw( + ArgumentError("required schema-bearing Parquet MAP-key leaf is missing")) + return + end + expected = _provenanceleaflogicaltype(binding.plan) + value isa expected && !_nestedwritekeyrecursive(value) || throw( + ArgumentError( + "schema-bearing Parquet MAP key contains $(typeof(value)); expected $expected")) + return +end + +function _nestedwritekeywitnessroot(binding::_ProvenanceStructBinding, value) + if ismissing(value) + _provenancerequired(binding.plan, binding.force_required) && throw(ArgumentError( + "required schema-bearing Parquet MAP-key struct is missing")) + return + end + value isa StructValue || throw(ArgumentError( + "schema-bearing Parquet MAP key requires StructValue")) + return +end + +function _nestedwritekeywitnessroot(binding::_ProvenanceListBinding, value) + if ismissing(value) + _provenancerequired(binding.plan, binding.force_required) && throw(ArgumentError( + "required schema-bearing Parquet MAP-key LIST is missing")) + return + end + value isa ListValue || throw(ArgumentError( + "schema-bearing Parquet MAP key requires ListValue")) + return +end + +function _nestedwritekeywitnessroot(binding::_ProvenanceMapBinding, value) + if ismissing(value) + _provenancerequired(binding.plan, binding.force_required) && throw(ArgumentError( + "required schema-bearing Parquet MAP-key MAP is missing")) + return + end + value isa MapValue || throw(ArgumentError( + "schema-bearing Parquet MAP key requires MapValue")) + return +end + +function _nestedwritekeywitnesssource(binding::_ProvenanceStructBinding, + index::Int) + index <= length(binding.children) || return nothing + return _provenancesource(binding.children[index]) +end + +function _nestedwritekeywitnesschild(binding::_ProvenanceStructBinding, + index::Int) + index <= length(binding.children) || return nothing + return binding.children[index] +end + +function _nestedwritekeywitnesslist(binding::_ProvenanceListBinding) + return binding.element +end + +function _nestedwritekeywitnessmapkey(binding::_ProvenanceMapBinding) + return binding.key +end + +function _nestedwritekeywitnessmapvalue(binding::_ProvenanceMapBinding) + return binding.value +end + +function _nestedwritekeystoredsource(binding::_ProvenanceStructBinding, + index::Int) + return _nestedwritekeywitnesssource(binding, index) +end + +function _provenancevalidatetop(table::Table, + semantic::_NestedSchemaPlan, bindings::Vector{_ProvenanceBinding}, + rows::Int, limits::Limits, topology::_NestedWriteTopologySnapshot, + budget::_LiveByteBudget) + table.rows == rows || throw(ArgumentError( + "table row count changed during schema-bearing write")) + columns = table.columns + values = Base.values(columns) + length(values) == length(bindings) == length(semantic.root.children) == + length(topology.names) || + throw(ArgumentError("table column count changed during schema-bearing write")) + for index in eachindex(bindings) + child = semantic.root.children[index] + topology.names[index] == child.source.element.name || + throw(ArgumentError( + "table column name or order changed during schema-bearing write")) + values[index] === _provenancesource(bindings[index]) || + throw(ArgumentError("table column identity changed during schema-bearing write")) + _provenancevalidate(bindings[index], rows, limits, budget) + end + return +end + +function _provenancetopology(table::Table, limits::Limits, + budget::_LiveByteBudget) + columns = table.columns + raw_names = keys(columns) + raw_values = Base.values(columns) + count = length(raw_values) + retained = _materializedsum(_materializedarraybytes(String, count), + _materializedarraybytes(AbstractVector, count)) + for raw in raw_names + retained = _materializedsum(retained, + _materializedsum(_MATERIALIZED_OBJECT_BYTES, + _writecolumnnamebytes(raw))) + end + _reserve!(budget, retained) + names = String[] + values = AbstractVector[] + sizehint!(names, count) + sizehint!(values, count) + for index in eachindex(raw_values) + value = raw_values[index] + value isa AbstractVector || throw(ArgumentError( + "schema-bearing Parquet columns must be vectors")) + push!(names, String(raw_names[index])) + push!(values, value) + end + return _nestedwritetopology(nothing, names, values, limits, budget; + retainedcharge=retained) +end + +function _provenancevalidateelements(elements::Vector{Metadata.SchemaElement}, + current::Vector{Metadata.SchemaElement}) + length(elements) == length(current) || throw(ArgumentError( + "stored Parquet SchemaElement count changed during write")) + for index in eachindex(elements, current) + _provenanceexact(elements[index], current[index]) || throw(ArgumentError( + "stored Parquet SchemaElement changed during write")) + end + return +end + +function _provenancebarrier!(table::Table, elements::Vector{Metadata.SchemaElement}, + schema::Schema, semantic::_NestedSchemaPlan, + bindings::Vector{_ProvenanceBinding}, rows::Int, limits::Limits, + topology::_NestedWriteTopologySnapshot, budget::_LiveByteBudget) + _provenancevalidateelements(elements, table.metadata.schema) + _provenancecompareschema(schema, table.schema, limits, budget) + _provenancevalidatetop(table, semantic, bindings, rows, limits, topology, + budget) + _nestedwritebarrier!(topology, nothing, budget) + return +end + +function _provenanceleafvalue(binding::_ProvenanceLeafBinding, index::Int) + values = binding.values + checkbounds(Bool, values, index) || throw(ArgumentError( + "schema-bearing physical leaf index exceeds its vector")) + return values[index] +end + +function _provenanceleafaccess(binding::_ProvenanceLeafBinding, index::Int) + return _nestedwriterowaccess(binding.values, index, binding.snapshot) +end + +function _provenancekeyroot(binding::_ProvenanceLeafBinding, value) + expected = _provenanceleaflogicaltype(binding.plan) + value isa expected || throw(ArgumentError( + "schema-bearing Parquet MAP key contains $(typeof(value)); expected $expected")) + return +end + +function _provenancekeyroot(::_ProvenanceStructBinding, value) + value isa StructValue || throw(ArgumentError( + "schema-bearing Parquet MAP key requires StructValue")) + return +end + +function _provenancekeyroot(::_ProvenanceListBinding, value) + value isa ListValue || throw(ArgumentError( + "schema-bearing Parquet MAP key requires ListValue")) + return +end + +function _provenancekeyroot(::_ProvenanceMapBinding, value) + value isa MapValue || throw(ArgumentError( + "schema-bearing Parquet MAP key requires MapValue")) + return +end + +function _provenancebindingvalue(binding::_ProvenanceBinding, index::Int) + values = _provenancesource(binding) + checkbounds(Bool, values, index) || throw(ArgumentError( + "schema-bearing nested value index exceeds its vector")) + return _nestedwriterowaccess(values, index, binding.snapshot) +end + +function _provenancetracesnapshotkey!(::Nothing, + ::_NestedWriteKeySnapshot) + return +end + +function _provenancetracesnapshotkey!(trace::_NestedWriteTrace, + snapshot::_NestedWriteKeySnapshot) + if trace.capturing + _nestedwritetracecapture!(trace, _NestedWriteTraceEvent( + _NESTED_WRITE_TRACE_KEY, Int64(0), Int64(0), Int64(0), Int64(0), + snapshot)) + return + end + expected = _nestedwritetracenext!(trace) + expected.kind == _NESTED_WRITE_TRACE_KEY && iszero(expected.a) && + iszero(expected.b) && iszero(expected.c) && iszero(expected.d) && + expected.value === snapshot || throw(ArgumentError( + "schema-bearing Parquet MAP-key replay changed its nested key topology")) + return +end + +function _provenancekeysnapshotpayload(element::Metadata.SchemaElement, + snapshot::_NestedWriteKeySnapshot, limits::Limits) + snapshot.kind in (_NESTED_WRITE_KEY_STRING, + _NESTED_WRITE_KEY_BYTES) || return _nestedwriteleafpayload(element, + snapshot.value, limits) + bytes = snapshot.value::Vector{UInt8} + kind = _logicalkind(element) + if kind in (:string, :enum) + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + isvalid(String, bytes) || throw(ArgumentError( + "schema-bearing Parquet MAP key contains invalid UTF-8")) + return Int64(length(bytes)) + elseif kind === :json + _validatejson(bytes, limits, ArgumentError) + return Int64(length(bytes)) + elseif kind === :bson + _validatebson(bytes, limits, ArgumentError) + return Int64(length(bytes)) + elseif element.type_ == Metadata.Type.BYTE_ARRAY + _checklimit(:string_bytes, length(bytes), limits.max_string_bytes) + return Int64(length(bytes)) + elseif element.type_ == Metadata.Type.FIXED_LEN_BYTE_ARRAY + return _nestedwritefixedpayload(element, bytes, limits) + end + throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot has incompatible physical bytes")) +end + +function _provenancekeysnapshotphysical(element::Metadata.SchemaElement, + snapshot::_NestedWriteKeySnapshot, limits::Limits) + snapshot.kind in (_NESTED_WRITE_KEY_STRING, + _NESTED_WRITE_KEY_BYTES) && return snapshot.value + return _nestedwritenormalizekeyphysical(element, snapshot.value, limits) +end + +function _provenancekeyleafcheck(binding::_ProvenanceLeafBinding, + snapshot::_NestedWriteKeySnapshot) + _nestedwritekeymissing(snapshot) && return + expected = _provenanceleaflogicaltype(binding.plan) + kind = snapshot.kind + if kind == _NESTED_WRITE_KEY_SCALAR + snapshot.value isa expected || throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot does not match its leaf binding")) + elseif kind == _NESTED_WRITE_KEY_STRING + snapshot.source_type <: AbstractString || throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot does not match its STRING binding")) + elseif kind == _NESTED_WRITE_KEY_BYTES + snapshot.source_type <: expected || throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot does not match its binary binding")) + else + throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot does not match its leaf binding")) + end + return +end + +function _provenanceshredkeyleaf!(context, + binding::_ProvenanceLeafBinding, + snapshot::_NestedWriteKeySnapshot, repetition::UInt64) + _provenancekeyleafcheck(binding, snapshot) + plan = binding.plan + leafindex = Int(first(plan.leaf_range)) + present = !_nestedwritekeymissing(snapshot) + _nestedwritetraceleaf!(context.trace, snapshot.value, present) + if !present + _provenancerequired(plan, binding.force_required) && throw( + ArgumentError( + "required schema-bearing Parquet MAP-key leaf is missing")) + _nestedwriterecord!(context, leafindex, repetition, + UInt64(plan.parent_definition), Int64(0), false, nothing) + return + end + element = plan.source.element + payload = _provenancekeysnapshotpayload(element, snapshot, context.limits) + if context isa _NestedWriteEmitContext + _nestedwritepreflightemit(context, leafindex, payload) + physical = _provenancekeysnapshotphysical(element, snapshot, + context.limits) + _nestedwriterecord!(context, leafindex, repetition, + UInt64(plan.present_definition), payload, true, physical) + else + _nestedwriterecord!(context, leafindex, repetition, + UInt64(plan.present_definition), payload, true, snapshot.value) + end + return +end + +function _provenancekeymissingcontainer!(context, + binding::_ProvenanceBinding, snapshot::_NestedWriteKeySnapshot, + repetition::UInt64, kind::UInt8) + _nestedwritekeymissing(snapshot) || return false + _provenancerequired(binding.plan, binding.force_required) && throw( + ArgumentError("required schema-bearing Parquet MAP key became null")) + if kind == _NESTED_WRITE_TRACE_STRUCT + _nestedwritetracestruct!(context.trace, missing, false, + length(binding.plan.children)) + else + _nestedwritetraceevent!(context.trace, kind, Int64(0), Int64(0), + Int64(0), Int64(0)) + end + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.parent_definition)) + return true +end + +function _provenancekeyreplayenter!(context, + binding::_ProvenanceLeafBinding, snapshot::_NestedWriteKeySnapshot, + repetition::UInt64, ::Int, + ::Union{Nothing,_ProvenanceKeyReplayFrame}) + _provenanceshredkeyleaf!(context, binding, snapshot, repetition) + return true +end + +function _provenancekeyreplayframe!(context, binding::_ProvenanceBinding, + snapshot::_NestedWriteKeySnapshot, repetition::UInt64, depth::Int, + parent::Union{Nothing,_ProvenanceKeyReplayFrame}) + trace = context.trace + trace === nothing && throw(AssertionError( + "schema-bearing MAP-key replay requires a writer trace")) + _nestedwritekeyframecharge!(trace) + try + return _ProvenanceKeyReplayFrame(parent, binding, snapshot, + repetition, depth, 0) + catch + _nestedwritekeyframefree!(trace) + rethrow() + end +end + +function _provenancekeyreplayenter!(context, + binding::_ProvenanceStructBinding, snapshot::_NestedWriteKeySnapshot, + repetition::UInt64, depth::Int, + parent::Union{Nothing,_ProvenanceKeyReplayFrame}) + _provenancekeymissingcontainer!(context, binding, snapshot, repetition, + _NESTED_WRITE_TRACE_STRUCT) && return true + snapshot.kind == _NESTED_WRITE_KEY_STRUCT || throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot is not a struct")) + length(snapshot.children) == length(binding.children) || throw( + ArgumentError( + "schema-bearing Parquet MAP-key struct changed its child count")) + _nestedwritetracestruct!(context.trace, snapshot, true, + length(binding.children)) + isempty(snapshot.children) && return true + return _provenancekeyreplayframe!(context, binding, snapshot, repetition, + depth, parent) +end + +function _provenancekeyreplayenter!(context, + binding::_ProvenanceListBinding, snapshot::_NestedWriteKeySnapshot, + repetition::UInt64, depth::Int, + parent::Union{Nothing,_ProvenanceKeyReplayFrame}) + _provenancekeymissingcontainer!(context, binding, snapshot, repetition, + _NESTED_WRITE_TRACE_LIST) && return true + snapshot.kind == _NESTED_WRITE_KEY_LIST || throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot is not a LIST")) + count = length(snapshot.children) + _nestedwritetraceevent!(context.trace, _NESTED_WRITE_TRACE_LIST, + Int64(1), Int64(count), Int64(snapshot.first), Int64(snapshot.last)) + if iszero(count) + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.present_definition)) + return true + end + return _provenancekeyreplayframe!(context, binding, snapshot, repetition, + depth, parent) +end + +function _provenancekeyreplayenter!(context, + binding::_ProvenanceMapBinding, snapshot::_NestedWriteKeySnapshot, + repetition::UInt64, depth::Int, + parent::Union{Nothing,_ProvenanceKeyReplayFrame}) + _provenancekeymissingcontainer!(context, binding, snapshot, repetition, + _NESTED_WRITE_TRACE_MAP) && return true + snapshot.kind == _NESTED_WRITE_KEY_MAP || throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot is not a MAP")) + iseven(length(snapshot.children)) || throw(ArgumentError( + "schema-bearing Parquet MAP-key snapshot has an incomplete entry")) + count = length(snapshot.children) ÷ 2 + _nestedwritetraceevent!(context.trace, _NESTED_WRITE_TRACE_MAP, + Int64(1), Int64(count), Int64(snapshot.first), Int64(snapshot.last)) + if iszero(count) + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.present_definition)) + return true + end + return _provenancekeyreplayframe!(context, binding, snapshot, repetition, + depth, parent) +end + +function _provenancekeyreplaynext(frame::_ProvenanceKeyReplayFrame) + while true + frame.position += 1 + snapshot = frame.snapshot + frame.position <= length(snapshot.children) || return nothing + binding = frame.binding + if binding isa _ProvenanceStructBinding + return binding.children[frame.position], + snapshot.children[frame.position], frame.repetition + elseif binding isa _ProvenanceListBinding + repetition = frame.position == 1 ? frame.repetition : + UInt64(binding.plan.repetition_level) + return binding.element, snapshot.children[frame.position], + repetition + elseif binding isa _ProvenanceMapBinding + entry = (frame.position + 1) ÷ 2 + repetition = entry == 1 ? frame.repetition : + UInt64(binding.plan.repetition_level) + child = snapshot.children[frame.position] + isodd(frame.position) && return binding.key, child, repetition + childbinding = binding.value + if childbinding === nothing + _nestedwritekeymissing(child) || throw(ArgumentError( + "key-only schema-bearing MAP-key snapshot contains a value")) + continue + end + return childbinding, child, repetition + end + throw(AssertionError("unknown schema-bearing MAP-key replay binding")) + end +end + +function _provenanceshredkey!(context, binding::_ProvenanceBinding, + snapshot::_NestedWriteKeySnapshot, repetition::UInt64) + trace = context.trace + started = _provenancekeyreplayenter!(context, binding, snapshot, + repetition, 1, nothing) + started === true && return + frame = started::_ProvenanceKeyReplayFrame + try + while true + next = _provenancekeyreplaynext(frame) + if next !== nothing + childbinding, childsnapshot, childrepetition = next + if frame.binding isa _ProvenanceMapBinding && + isodd(frame.position) + _provenancetracesnapshotkey!(trace, childsnapshot) + end + depth = _nestedwritekeynextdepth(frame.depth, context.limits) + _checklimit(:metadata_depth, depth, + context.limits.max_metadata_depth) + started = _provenancekeyreplayenter!(context, childbinding, + childsnapshot, childrepetition, depth, frame) + started === true || (frame = started) + continue + end + parent = frame.parent + _nestedwritekeyframefree!(trace) + if parent === nothing + frame = nothing + return + end + frame = parent + end + finally + while frame !== nothing + parent = frame.parent + _nestedwritekeyframefree!(trace) + frame = parent + end + end +end + +function _provenanceshred!(context::_NestedWriteCountContext, + binding::_ProvenanceLeafBinding, index::Int, repetition::UInt64) + plan = binding.plan + value, row_witness = _provenanceleafaccess(binding, index) + _nestedwriterowcheck(row_witness) + leafindex = Int(first(plan.leaf_range)) + present = !ismissing(value) + _nestedwritetraceleaf!(context.trace, value, present) + if !present + _provenancerequired(plan, binding.force_required) && throw(ArgumentError( + "required physical leaf $(repr(join(plan.source.path, "."))) is missing")) + _nestedwriterecord!(context, leafindex, repetition, + UInt64(plan.parent_definition), Int64(0), false, nothing) + return + end + expected = _provenanceleaflogicaltype(plan) + value isa expected || throw(ArgumentError( + "physical leaf $(repr(join(plan.source.path, "."))) contains " * + "$(typeof(value)); expected $expected")) + payload = _nestedwriteleafpayload(plan.source.element, value, context.limits) + _nestedwriterecord!(context, leafindex, repetition, + UInt64(plan.present_definition), payload, true, value) + _nestedwriterowcheck(row_witness) + return +end + +function _provenanceshred!(context::_NestedWriteEmitContext, + binding::_ProvenanceLeafBinding, index::Int, repetition::UInt64) + plan = binding.plan + value, row_witness = _provenanceleafaccess(binding, index) + _nestedwriterowcheck(row_witness) + leafindex = Int(first(plan.leaf_range)) + present = !ismissing(value) + _nestedwritetraceleaf!(context.trace, value, present) + if !present + _provenancerequired(plan, binding.force_required) && throw(ArgumentError( + "required physical leaf $(repr(join(plan.source.path, "."))) is missing")) + _nestedwriterecord!(context, leafindex, repetition, + UInt64(plan.parent_definition), Int64(0), false, nothing) + return + end + expected = _provenanceleaflogicaltype(plan) + value isa expected || throw(ArgumentError( + "physical leaf $(repr(join(plan.source.path, "."))) contains " * + "$(typeof(value)); expected $expected")) + element = plan.source.element + payload = _nestedwriteleafpayload(element, value, context.limits) + _nestedwritepreflightemit(context, leafindex, payload) + physical = _nestedwritenormalizephysical(element, value, context.limits) + _nestedwriterecord!(context, leafindex, repetition, + UInt64(plan.present_definition), payload, true, physical) + _nestedwriterowcheck(row_witness) + return +end + +function _provenanceshredenter!(context, binding::_ProvenanceLeafBinding, + index::Int, repetition::UInt64, + ::Union{Nothing,_ProvenanceShredFrame}) + _provenanceshred!(context, binding, index, repetition) + return true +end + +function _provenanceshredframe(context, binding::_ProvenanceBinding, + repetition::UInt64, childindex::Int, position::Int, last::Int, + witness::_NestedWriteRowWitness, parent) + trace = context.trace + trace === nothing && throw(AssertionError( + "schema-bearing traversal requires a writer trace")) + _reserveobjects!(trace.budget) + try + return _ProvenanceShredFrame(parent, binding, repetition, childindex, + position, last, witness) + catch + _release!(trace.budget, _MATERIALIZED_OBJECT_BYTES) + rethrow() + end +end + +function _provenanceshredframefree!(context) + trace = context.trace + trace === nothing && throw(AssertionError( + "schema-bearing traversal requires a writer trace")) + _release!(trace.budget, _MATERIALIZED_OBJECT_BYTES) + return +end + +function _provenanceshredframesfree!(context, count::Int) + iszero(count) && return + trace = context.trace + trace === nothing && throw(AssertionError( + "schema-bearing traversal requires a writer trace")) + _release!(trace.budget, + _materializedproduct(count, _MATERIALIZED_OBJECT_BYTES)) + return +end + +function _provenancestructindex(binding::_ProvenanceStructBinding, index::Int) + witness = _nestedwriterowwitness(binding.snapshot, index) + witness === nothing && throw(AssertionError( + "schema-bearing struct has no row witness")) + return witness.present ? witness.last : 0, witness +end + +function _provenanceshredenter!(context, + binding::_ProvenanceStructBinding, index::Int, + repetition::UInt64, parent) + childindex, row_witness = _provenancestructindex(binding, index) + _nestedwriterowcheck(row_witness) + present = !iszero(childindex) + _nestedwritetracestruct!(context.trace, binding.values, present, + length(binding.children)) + if iszero(childindex) + _provenancerequired(binding.plan, binding.force_required) && + throw(ArgumentError("required struct became null during schema-bearing write")) + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.parent_definition)) + return true + end + values = binding.values + childcount = length(values.children) + length(binding.children) == childcount || throw(ArgumentError( + "schema-bearing struct changed its child count during access")) + iszero(childcount) && return true + return _provenanceshredframe(context, binding, repetition, childindex, 0, + childcount, row_witness, parent) +end + +function _provenancelistspan(binding::_ProvenanceListBinding, index::Int) + witness = _nestedwriterowwitness(binding.snapshot, index) + witness === nothing && throw(AssertionError( + "schema-bearing list has no row witness")) + witness.present || return 0, -1, witness + return witness.first, witness.last, witness +end + +function _provenanceshredenter!(context, binding::_ProvenanceListBinding, + index::Int, repetition::UInt64, parent) + firstentry, lastentry, row_witness = _provenancelistspan(binding, index) + _nestedwriterowcheck(row_witness) + present = !iszero(firstentry) + count = present ? max(lastentry - firstentry + 1, 0) : 0 + _nestedwritetraceevent!(context.trace, _NESTED_WRITE_TRACE_LIST, + present ? Int64(1) : Int64(0), Int64(count), Int64(firstentry), + Int64(lastentry)) + if iszero(firstentry) + _provenancerequired(binding.plan, binding.force_required) && + throw(ArgumentError("required list became null during schema-bearing write")) + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.parent_definition)) + return true + elseif firstentry > lastentry + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.present_definition)) + return true + end + return _provenanceshredframe(context, binding, repetition, 0, + firstentry - 1, lastentry, row_witness, parent) +end + +function _provenancemapspan(binding::_ProvenanceMapBinding, index::Int) + witness = _nestedwriterowwitness(binding.snapshot, index) + witness === nothing && throw(AssertionError( + "schema-bearing map has no row witness")) + witness.present || return 0, -1, witness + return witness.first, witness.last, witness +end + +function _provenanceshredenter!(context, binding::_ProvenanceMapBinding, + index::Int, repetition::UInt64, parent) + firstentry, lastentry, row_witness = _provenancemapspan(binding, index) + _nestedwriterowcheck(row_witness) + present = !iszero(firstentry) + count = present ? max(lastentry - firstentry + 1, 0) : 0 + _nestedwritetraceevent!(context.trace, _NESTED_WRITE_TRACE_MAP, + present ? Int64(1) : Int64(0), Int64(count), Int64(firstentry), + Int64(lastentry)) + if iszero(firstentry) + _provenancerequired(binding.plan, binding.force_required) && + throw(ArgumentError("required map became null during schema-bearing write")) + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.parent_definition)) + return true + elseif firstentry > lastentry + _nestedwritemarker!(context, binding.plan.leaf_range, repetition, + UInt64(binding.plan.present_definition)) + return true + end + return _provenanceshredframe(context, binding, repetition, 0, + firstentry - 1, lastentry, row_witness, parent) +end + +function _provenanceshrednext(context, frame::_ProvenanceShredFrame) + binding = frame.binding + if binding isa _ProvenanceStructBinding + frame.position += 1 + frame.position <= frame.last || return nothing + _nestedwriterowcheck(frame.witness) + values = binding.values + length(values.children) == frame.last || throw(ArgumentError( + "schema-bearing struct changed its child count during access")) + child = binding.children[frame.position] + values.children[frame.position] === _provenancesource(child) || throw( + ArgumentError( + "schema-bearing struct changed child identity during access")) + return child, frame.childindex, frame.repetition + elseif binding isa _ProvenanceListBinding + frame.position += 1 + frame.position <= frame.last || return nothing + _nestedwriterowcheck(frame.witness) + repetition = frame.position == frame.witness.first ? + frame.repetition : UInt64(binding.plan.repetition_level) + return binding.element, frame.position, repetition + elseif binding isa _ProvenanceMapBinding + while true + frame.position += 1 + frame.position <= frame.last || return nothing + _nestedwriterowcheck(frame.witness) + repetition = frame.position == frame.witness.first ? + frame.repetition : UInt64(binding.plan.repetition_level) + keyvalue, key_witness = _provenancebindingvalue(binding.key, + frame.position) + _nestedwriterowcheck(key_witness) + _nestedwriterowcheck(frame.witness) + ismissing(keyvalue) && throw(ArgumentError( + "map $(repr(join(binding.plan.source.path, "."))) contains a null key")) + expected = _nestedwritetracekey!(context.trace, keyvalue, nothing, + context.limits, binding.key, key_witness) + _nestedwriterowcheck(frame.witness) + _provenanceshredkey!(context, binding.key, expected, repetition) + _nestedwriterowcheck(frame.witness) + binding.value === nothing && continue + return something(binding.value), frame.position, repetition + end + end + throw(AssertionError("unknown schema-bearing shred frame")) +end + +function _provenanceshrediterative!(context, + binding::_ProvenanceBinding, index::Int, repetition::UInt64) + started = _provenanceshredenter!(context, binding, index, repetition, + nothing) + started === true && return + current::Union{Nothing,_ProvenanceShredFrame} = + started::_ProvenanceShredFrame + activeframes = 1 + try + while true + next = _provenanceshrednext(context, current) + if next !== nothing + child, childindex, childrepetition = next + started = _provenanceshredenter!(context, child, childindex, + childrepetition, current) + if started !== true + activeframes += 1 + current = started::_ProvenanceShredFrame + end + continue + end + parent = current.parent + _provenanceshredframefree!(context) + activeframes -= 1 + if parent === nothing + current = nothing + return + end + current = parent::_ProvenanceShredFrame + end + finally + _provenanceshredframesfree!(context, activeframes) + end +end + +function _provenanceshred!(context, binding::_ProvenanceStructBinding, + index::Int, repetition::UInt64) + return _provenanceshrediterative!(context, binding, index, repetition) +end + +function _provenanceshred!(context, binding::_ProvenanceListBinding, + index::Int, repetition::UInt64) + return _provenanceshrediterative!(context, binding, index, repetition) +end + +function _provenanceshred!(context, binding::_ProvenanceMapBinding, + index::Int, repetition::UInt64) + return _provenanceshrediterative!(context, binding, index, repetition) +end + +function _provenancepass!(context, bindings::Vector{_ProvenanceBinding}, rows::Int) + for row in 1:rows + for binding in bindings + _provenanceshred!(context, binding, row, UInt64(0)) + end + _nestedwritefinishrow!(context, eachindex(context isa + _NestedWriteCountContext ? context.counts : context.builders), row) + end + return +end + +function _provenanceschemawalkframe(node::SchemaNode, parent, depth::Int, + limits::Limits, budget::_LiveByteBudget) + _checklimit(:metadata_depth, depth, limits.max_metadata_depth) + _checklimit(:container_elements, length(node.children), + limits.max_container_elements) + isempty(node.children) && return nothing + _reserveobjects!(budget) + return _ProvenanceSchemaWalkFrame(parent, node, depth, 0) +end + +function _provenanceschemawalknext(frame::_ProvenanceSchemaWalkFrame) + frame.position += 1 + return frame.node.children[frame.position] +end + +function _provenancefragmentcount(node::SchemaNode, limits::Limits, + budget::_LiveByteBudget) + count = Int64(1) + _checklimit(:container_elements, count, limits.max_container_elements) + depth = _nestedwritedepthadd(length(node.path), 1, limits) + current::Union{Nothing,_ProvenanceSchemaWalkFrame} = + _provenanceschemawalkframe(node, nothing, depth, limits, budget) + activeframes = current === nothing ? 0 : 1 + try + current === nothing && return Int(count) + while true + if current.position == length(current.node.children) + parent = current.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + activeframes -= 1 + if parent === nothing + current = nothing + return Int(count) + end + current = parent::_ProvenanceSchemaWalkFrame + continue + end + child = _provenanceschemawalknext(current) + count = try + Base.checked_add(count, Int64(1)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, count, + limits.max_container_elements) + childdepth = _nestedwritedepthadd(current.depth, 1, limits) + childframe = _provenanceschemawalkframe(child, current, childdepth, + limits, budget) + if childframe !== nothing + activeframes += 1 + current = childframe::_ProvenanceSchemaWalkFrame + end + end + finally + _provenanceframesrelease!(budget, activeframes) + end +end + +function _provenancefragmentappend!(elements::Vector{Metadata.SchemaElement}, + node::SchemaNode, limits::Limits, budget::_LiveByteBudget) + push!(elements, node.element) + depth = _nestedwritedepthadd(length(node.path), 1, limits) + current::Union{Nothing,_ProvenanceSchemaWalkFrame} = + _provenanceschemawalkframe(node, nothing, depth, limits, budget) + activeframes = current === nothing ? 0 : 1 + try + current === nothing && return + while true + if current.position == length(current.node.children) + parent = current.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + activeframes -= 1 + if parent === nothing + current = nothing + return + end + current = parent::_ProvenanceSchemaWalkFrame + continue + end + child = _provenanceschemawalknext(current) + push!(elements, child.element) + childdepth = _nestedwritedepthadd(current.depth, 1, limits) + childframe = _provenanceschemawalkframe(child, current, childdepth, + limits, budget) + if childframe !== nothing + activeframes += 1 + current = childframe::_ProvenanceSchemaWalkFrame + end + end + finally + _provenanceframesrelease!(budget, activeframes) + end +end + +function _provenancefragment(node::SchemaNode, limits::Limits, + budget::_LiveByteBudget) + count = _provenancefragmentcount(node, limits, budget) + _reservearray!(budget, Metadata.SchemaElement, count) + elements = Metadata.SchemaElement[] + sizehint!(elements, count) + _provenancefragmentappend!(elements, node, limits, budget) + return elements +end + +function _provenancefinishfields(semantic::_NestedSchemaPlan, + builders::Vector{_NestedWriteLeafBuilder}, rows::Int, limits::Limits, + budget::_LiveByteBudget) + children = semantic.root.children + _reservearray!(budget, WriteFieldPlan, length(children)) + fields = WriteFieldPlan[] + sizehint!(fields, length(children)) + for child in children + range = _nestedleafrange(child) + isempty(range) && throw(ArgumentError( + "schema-bearing writes do not support zero-leaf fields")) + fragment = _provenancefragment(child.source, limits, budget) + _reservearray!(budget, WriteColumn, length(range)) + leaves = WriteColumn[] + sizehint!(leaves, length(range)) + for rawindex in range + index = Int(rawindex) + push!(leaves, _nestedwritecolumn(builders[index], + semantic.leaves[index], rows, budget)) + end + _reserveobjects!(budget) + push!(fields, WriteFieldPlan(fragment, leaves)) + end + return fields +end + +function _provenancewritefields(table::Table, limits::Limits, + budget::_LiveByteBudget, encoding, dictionary::Bool) + start = _budgetused(budget) + try + rows = table.rows + rows >= 0 || throw(ArgumentError("Parquet row count must be nonnegative")) + _checklimit(:container_elements, rows, limits.max_container_elements) + elements, schema = _provenancefreshschema(table, limits, budget) + semantic = _nestedplan(schema; limits=limits, budget=budget) + _provenancerejectleafless(semantic.root, budget) + topology = _provenancetopology(table, limits, budget) + bindings = _provenancebindings(table, semantic, limits, budget, + topology) + _provenancebarrier!(table, elements, schema, semantic, bindings, rows, + limits, topology, budget) + choices, choicecharge = _preflightwriteencoding(semantic, encoding, + dictionary, budget) + _provenancebarrier!(table, elements, schema, semantic, bindings, rows, + limits, topology, budget) + trace = _nestedwritetrace(budget, topology) + counts = _nestedwritecounts(length(semantic.leaves), budget) + countcontext, boundarycharge = _nestedwritecountcontext(counts, + semantic, rows, limits, budget, trace) + _provenancebarrier!(table, elements, schema, semantic, bindings, rows, + limits, topology, budget) + _provenancepass!(countcontext, bindings, rows) + _provenancebarrier!(table, elements, schema, semantic, bindings, rows, + limits, topology, budget) + _nestedwritevalidateboundaries(countcontext, rows) + _nestedwritepreflightrows(countcontext, choices) + _release!(budget, choicecharge) + choices = nothing + builders = _nestedwritebuilders(counts, semantic, budget) + _provenancebarrier!(table, elements, schema, semantic, bindings, rows, + limits, topology, budget) + emitcontext = _NestedWriteEmitContext(builders, counts, limits, + countcontext, trace) + _nestedwritetracecompare!(trace) + _provenancepass!(emitcontext, bindings, rows) + _nestedwritetracefinishcompare!(trace) + _provenancebarrier!(table, elements, schema, semantic, bindings, rows, + limits, topology, budget) + _nestedwritevalidatebuilders(builders, counts, semantic, rows) + fields = _provenancefinishfields(semantic, builders, rows, limits, + budget) + _provenancebarrier!(table, elements, schema, semantic, bindings, rows, + limits, topology, budget) + _release!(budget, boundarycharge) + countcontext = nothing + emitcontext = nothing + _nestedwritetracerelease!(trace) + trace = nothing + _nestedwritetopologyrelease!(topology, budget) + topology = nothing + _reserveobjects!(budget) + return _ProvenanceWriteFields(fields, elements, schema), rows + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _provenancevalidatefieldnode(field::WriteFieldPlan, + node::SchemaNode, index::Int) + index <= length(field.schema) || throw(ArgumentError( + "schema-bearing writer field fragment is truncated")) + _provenanceexact(field.schema[index], node.element) || + throw(ArgumentError("schema-bearing writer SchemaElement changed")) + return index + 1 +end + +function _provenancevalidatefieldfragment(field::WriteFieldPlan, + node::SchemaNode, index::Int, limits::Limits, + budget::_LiveByteBudget) + next = _provenancevalidatefieldnode(field, node, index) + depth = _nestedwritedepthadd(length(node.path), 1, limits) + current::Union{Nothing,_ProvenanceSchemaWalkFrame} = + _provenanceschemawalkframe(node, nothing, depth, limits, budget) + activeframes = current === nothing ? 0 : 1 + try + current === nothing && return next + while true + if current.position == length(current.node.children) + parent = current.parent + _release!(budget, _MATERIALIZED_OBJECT_BYTES) + activeframes -= 1 + if parent === nothing + current = nothing + return next + end + current = parent::_ProvenanceSchemaWalkFrame + continue + end + child = _provenanceschemawalknext(current) + next = _provenancevalidatefieldnode(field, child, next) + childdepth = _nestedwritedepthadd(current.depth, 1, limits) + childframe = _provenanceschemawalkframe(child, current, childdepth, + limits, budget) + if childframe !== nothing + activeframes += 1 + current = childframe::_ProvenanceSchemaWalkFrame + end + end + finally + _provenanceframesrelease!(budget, activeframes) + end +end + +function _provenancevalidatefieldfragments(fields::_ProvenanceWriteFields, + limits::Limits, budget::_LiveByteBudget) + length(fields.fields) == length(fields.schema.root.children) || + throw(ArgumentError("schema-bearing writer field count changed")) + for (field, node) in zip(fields.fields, fields.schema.root.children) + next = _provenancevalidatefieldfragment(field, node, 1, limits, + budget) + next == length(field.schema) + 1 || throw(ArgumentError( + "schema-bearing writer field fragment has trailing elements")) + end + return +end + +function _writeplanleaves(fields::_ProvenanceWriteFields, schema::Schema, + rows::Int, limits::Limits, budget::_LiveByteBudget) + schema === fields.schema || throw(ArgumentError( + "schema-bearing writer did not use its operation-owned schema")) + return _writeplanleaves(fields.fields, schema, rows, limits, budget) +end + +function _writeplan(fields::_ProvenanceWriteFields, rows::Int, limits::Limits, + budget::_LiveByteBudget) + start = _budgetused(budget) + try + rows >= 0 || throw(ArgumentError( + "Parquet row count must be nonnegative")) + _provenancevalidatefieldfragments(fields, limits, budget) + leaves, leafcharge = _writeplanleaves(fields.fields, fields.schema, + rows, limits, budget) + _reservearray!(budget, WriteRowGroupPlan, iszero(rows) ? 0 : 1) + _reserveobjects!(budget, 2) + rowgroups = iszero(rows) ? WriteRowGroupPlan[] : + WriteRowGroupPlan[WriteRowGroupPlan(rows, leaves)] + plan = WritePlan(fields.elements, fields.schema, rows, rowgroups) + iszero(rows) && _release!(budget, leafcharge) + return plan + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end diff --git a/src/write_splitting.jl b/src/write_splitting.jl new file mode 100644 index 0000000..8dcddb3 --- /dev/null +++ b/src/write_splitting.jl @@ -0,0 +1,814 @@ +# Row-boundary ownership and slicing for row groups and data pages. + +struct _WritePageCapacityError <: Exception + resource::Symbol + requested::Int64 + maximum::Int64 +end + +function _writecheckedint64(value::Integer, resource::Symbol, maximum::Int64) + value >= 0 || throw(ArgumentError("$resource must be nonnegative")) + value <= typemax(Int64) || throw(LimitError(resource, typemax(Int64), maximum)) + return Int64(value) +end + +function _writedensecopy(values::AbstractVector, budget::_LiveByteBudget) + T = Base.nonmissingtype(eltype(values)) + T === Missing && throw(ArgumentError( + "a physical Parquet leaf cannot contain only missing dense values")) + count = 0 + for value in values + ismissing(value) || (count = Base.checked_add(count, 1)) + end + charge = _reservearray!(budget, T, count) + output = T[] + sizehint!(output, count) + for value in values + ismissing(value) && continue + push!(output, convert(T, value)) + end + return output, charge +end + +function _writenormalizedense(column::WriteColumn, rows::Int, + budget::_LiveByteBudget) + definitions = column.definitions + values = column.values + charge = Int64(0) + if definitions === nothing && !iszero(column.max_definition_level) + iszero(column.max_repetition_level) || throw(ArgumentError( + "a repeated writer leaf with definition levels must provide its level stream")) + column.max_definition_level == 1 || throw(ArgumentError( + "a nested writer leaf with definition levels must provide its level stream")) + length(values) == rows || throw(ArgumentError( + "optional writer leaf input does not match the table row count")) + definitioncharge = _reservearray!(budget, UInt64, rows) + dense, densecharge = try + _writedensecopy(values, budget) + catch + _release!(budget, definitioncharge) + rethrow() + end + definitions = Vector{UInt64}(undef, rows) + index = 0 + for value in values + index += 1 + definitions[index] = ismissing(value) ? UInt64(0) : UInt64(1) + end + values = dense + charge = _materializedsum(definitioncharge, densecharge) + elseif Missing <: eltype(values) || !(values isa Vector) + dense, densecharge = _writedensecopy(values, budget) + if length(dense) != length(values) + _release!(budget, densecharge) + throw(ArgumentError( + "writer dense physical values contain missing")) + end + values = dense + charge = densecharge + else + for value in values + ismissing(value) && throw(ArgumentError( + "writer dense physical values contain missing")) + end + end + values === column.values && definitions === column.definitions && return column, charge + normalized = WriteColumn(column.name, values, column.physical, + column.type_length, column.optional, column.logical, column.converted, + column.path, column.repetitions, definitions, + column.max_repetition_level, column.max_definition_level, column.rows, + column.schema) + return normalized, charge +end + +function _writeentryoffsets(column::WriteColumn, rows::Int) + entries = _columnentrycount(column) + output = Vector{Int64}(undef, rows + 1) + if column.repetitions === nothing + entries == rows || throw(ArgumentError( + "flat writer leaf has $entries level entries for $rows rows")) + for row in 0:rows + output[row + 1] = Int64(row) + end + return output + end + repetitions = column.repetitions + length(repetitions) == entries || throw(ArgumentError( + "writer repetition stream length changed")) + row = 0 + for index in eachindex(repetitions) + level = repetitions[index] + level <= UInt64(column.max_repetition_level) || throw(ArgumentError( + "writer repetition level exceeds its schema maximum")) + if iszero(level) + row += 1 + row <= rows || throw(ArgumentError( + "writer leaf has more row boundaries than the table")) + output[row] = Int64(index - firstindex(repetitions)) + end + end + row == rows || throw(ArgumentError( + "writer leaf has $row row boundaries for $rows rows")) + output[rows + 1] = Int64(entries) + return output +end + +function _writedenseoffsets(column::WriteColumn, entries::Vector{Int64}) + rows = length(entries) - 1 + output = Vector{Int64}(undef, rows + 1) + output[1] = Int64(0) + dense = Int64(0) + definitions = column.definitions + if definitions === nothing + iszero(column.max_definition_level) || throw(ArgumentError( + "writer leaf is missing its definition stream")) + for row in 1:rows + dense = Base.checked_add(dense, entries[row + 1] - entries[row]) + output[row + 1] = dense + end + else + length(definitions) == last(entries) || throw(ArgumentError( + "writer definition stream length changed")) + maximum = UInt64(column.max_definition_level) + position = 0 + for row in 1:rows + stop = Int(entries[row + 1]) + while position < stop + position += 1 + definition = definitions[position] + definition <= maximum || throw(ArgumentError( + "writer definition level exceeds its schema maximum")) + definition == maximum && (dense = Base.checked_add(dense, 1)) + end + output[row + 1] = dense + end + end + dense == length(column.values) || throw(ArgumentError( + "writer dense-value count does not match its definition stream")) + return output +end + +function _writepayloadfixedwidth(column::WriteColumn) + column.physical in (Metadata.Type.INT32, Metadata.Type.FLOAT) && return Int64(4) + column.physical in (Metadata.Type.INT64, Metadata.Type.DOUBLE) && return Int64(8) + if column.physical == Metadata.Type.FIXED_LEN_BYTE_ARRAY + width = column.type_length + width === nothing && throw(ArgumentError( + "fixed byte-array writer leaf has no width")) + return Int64(width) + end + return nothing +end + +function _writepayloadoffsets(column::WriteColumn, dense::Vector{Int64}, + limits::Limits) + rows = length(dense) - 1 + output = Vector{Int64}(undef, rows + 1) + physical = column.physical + if physical == Metadata.Type.BOOLEAN + for row in 0:rows + output[row + 1] = cld(dense[row + 1], Int64(8)) + end + return output + end + width = _writepayloadfixedwidth(column) + if width !== nothing + for row in 0:rows + output[row + 1] = Base.checked_mul(dense[row + 1], width) + end + return output + end + physical == Metadata.Type.BYTE_ARRAY || throw(ArgumentError( + "unsupported writer physical type $physical")) + output[1] = Int64(0) + bytes = Int64(0) + position = 0 + for row in 1:rows + stop = Int(dense[row + 1]) + while position < stop + position += 1 + value = column.values[position] + payload = value isa AbstractString ? ncodeunits(value) : length(value) + _checklimit(:string_bytes, payload, limits.max_string_bytes) + payload <= typemax(Int32) || throw(ArgumentError( + "byte array exceeds Int32 length")) + bytes = Base.checked_add(bytes, Base.checked_add(Int64(4), + Int64(payload))) + end + output[row + 1] = bytes + end + return output +end + +function _writevalidateprefixes(column::WriteColumn, rows::Int, + entries::Vector{Int64}, dense::Vector{Int64}, payload::Vector{Int64}) + length(entries) == rows + 1 == length(dense) == length(payload) || + throw(AssertionError("writer row-boundary prefix lengths differ")) + first(entries) == first(dense) == first(payload) == 0 || + throw(AssertionError("writer row-boundary prefixes do not start at zero")) + issorted(entries) && issorted(dense) && issorted(payload) || + throw(AssertionError("writer row-boundary prefixes are not monotonic")) + last(entries) == _columnentrycount(column) || throw(AssertionError( + "writer entry prefix has the wrong terminal count")) + last(dense) == length(column.values) || throw(AssertionError( + "writer dense prefix has the wrong terminal count")) + last(payload) == _writerrawpayloadbytes(column) || throw(AssertionError( + "writer payload prefix has the wrong terminal byte count")) + for row in 1:rows + entries[row + 1] > entries[row] || throw(ArgumentError( + "every top-level row must add a level entry to every leaf")) + start = Int(entries[row]) + 1 + repetitions = column.repetitions + repetitions === nothing || iszero(repetitions[start]) || + throw(ArgumentError("writer data-page row boundary has nonzero repetition")) + end + return +end + +function _writeprepareleaf(node::SchemaNode, column::WriteColumn, rows::Int, + limits::Limits, budget::_LiveByteBudget) + start = _budgetused(budget) + try + normalized, densecharge = _writenormalizedense(column, rows, budget) + prefixcharge = _materializedproduct(3, + _materializedarraybytes(Int64, rows + 1)) + _reserve!(budget, prefixcharge) + entries = _writeentryoffsets(normalized, rows) + dense = _writedenseoffsets(normalized, entries) + payload = _writepayloadoffsets(normalized, dense, limits) + _writevalidateprefixes(normalized, rows, entries, dense, payload) + return normalized, entries, dense, payload, + _materializedsum(densecharge, prefixcharge) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writerowgroupsize(value, rows::Int) + value === nothing && return max(rows, 1) + value isa Integer && !(value isa Bool) || throw(ArgumentError( + "rowgroupsize must be a positive integer or nothing")) + value > 0 || throw(ArgumentError( + "rowgroupsize must be a positive integer or nothing")) + value <= typemax(Int) || throw(ArgumentError( + "rowgroupsize exceeds the Julia index range")) + return Int(value) +end + +function _writepagesize(value) + value === nothing && return nothing + value isa Integer && !(value isa Bool) || throw(ArgumentError( + "pagesize must be a positive integer or nothing")) + value > 0 || throw(ArgumentError( + "pagesize must be a positive integer or nothing")) + value <= typemax(Int64) || throw(ArgumentError( + "pagesize exceeds Int64")) + return Int64(value) +end + +function _writeslicelevels(levels::Union{Nothing,AbstractVector{UInt64}}, first::Int, + last::Int, budget::_LiveByteBudget) + levels === nothing && return nothing, Int64(0) + _reserveobjects!(budget) + return @view(levels[(first + 1):last]), _MATERIALIZED_OBJECT_BYTES +end + +function _writesliceprefixvalues(prefix::Vector{Int64}, firstrow::Int, + lastrow::Int) + count = lastrow - firstrow + 1 + base = prefix[firstrow + 1] + output = Vector{Int64}(undef, count) + for index in 0:(count - 1) + output[index + 1] = prefix[firstrow + index + 1] - base + end + return output +end + +function _writesliceprefix(prefix::Vector{Int64}, firstrow::Int, lastrow::Int, + budget::_LiveByteBudget) + count = lastrow - firstrow + 1 + charge = _reservearray!(budget, Int64, count) + output = _writesliceprefixvalues(prefix, firstrow, lastrow) + return output, charge +end + +function _writesliceleaf(leaf::WriteLeafPlan, firstrow::Int, lastrow::Int, + limits::Limits, budget::_LiveByteBudget) + 0 <= firstrow <= lastrow <= leaf.column.rows || throw(ArgumentError( + "writer row-group slice is outside its leaf")) + start = _budgetused(budget) + try + entrystart = Int(leaf.entry_offsets[firstrow + 1]) + entrystop = Int(leaf.entry_offsets[lastrow + 1]) + densestart = Int(leaf.dense_offsets[firstrow + 1]) + densestop = Int(leaf.dense_offsets[lastrow + 1]) + repetitions, repetitioncharge = _writeslicelevels( + leaf.column.repetitions, entrystart, entrystop, budget) + definitions, definitioncharge = _writeslicelevels( + leaf.column.definitions, entrystart, entrystop, budget) + repetitioncharge >= 0 && definitioncharge >= 0 || throw(AssertionError( + "writer level slice charge is negative")) + _reserveobjects!(budget, 2) + values = @view leaf.column.values[(densestart + 1):densestop] + entries, entrycharge = _writesliceprefix(leaf.entry_offsets, + firstrow, lastrow, budget) + dense, densecharge = _writesliceprefix(leaf.dense_offsets, + firstrow, lastrow, budget) + payloadcharge = _reservearray!(budget, Int64, + lastrow - firstrow + 1) + entrycharge >= 0 && densecharge >= 0 && payloadcharge >= 0 || + throw(AssertionError("writer prefix slice charge is negative")) + rows = lastrow - firstrow + source = leaf.column + payload = source.physical == Metadata.Type.BOOLEAN ? + _writepayloadoffsets(source, dense, limits) : + _writesliceprefixvalues(leaf.payload_offsets, firstrow, lastrow) + column = WriteColumn(source.name, values, source.physical, + source.type_length, source.optional, source.logical, + source.converted, leaf.path, repetitions, definitions, + source.max_repetition_level, source.max_definition_level, rows, + source.schema) + _writevalidateprefixes(column, rows, entries, dense, payload) + return WriteLeafPlan(leaf.ordinal, leaf.path, column, entries, + dense, payload) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writegroupcount(rows::Int, size::Int) + iszero(rows) && return 0 + return Base.checked_add(fld(rows - 1, size), 1) +end + +function _splitwriteplan(plan::WritePlan, rowgroupsize, limits::Limits, + budget::_LiveByteBudget) + size = _writerowgroupsize(rowgroupsize, plan.rows) + isempty(plan.rowgroups) && return plan + length(plan.rowgroups) == 1 || throw(AssertionError( + "writer plan was split more than once")) + source = only(plan.rowgroups) + source.rows == plan.rows || throw(AssertionError( + "writer source row group does not own every table row")) + size >= plan.rows && return plan + count = try + _writegroupcount(plan.rows, size) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, count, limits.max_container_elements) + leafcount = try + Base.checked_mul(count, length(source.leaves)) + catch err + err isa OverflowError || rethrow() + throw(LimitError(:container_elements, typemax(Int64), + limits.max_container_elements)) + end + _checklimit(:container_elements, leafcount, limits.max_container_elements) + start = _budgetused(budget) + try + _reservearray!(budget, WriteRowGroupPlan, count) + _reserveobjects!(budget, count + 1) + groups = WriteRowGroupPlan[] + sizehint!(groups, count) + firstrow = 0 + while firstrow < plan.rows + lastrow = min(Base.checked_add(firstrow, size), plan.rows) + _reservearray!(budget, WriteLeafPlan, length(source.leaves)) + leaves = WriteLeafPlan[] + sizehint!(leaves, length(source.leaves)) + for leaf in source.leaves + push!(leaves, _writesliceleaf(leaf, firstrow, lastrow, limits, + budget)) + end + push!(groups, WriteRowGroupPlan(lastrow - firstrow, leaves)) + firstrow = lastrow + end + length(groups) == count || throw(AssertionError( + "writer row-group split count changed")) + output = WritePlan(plan.elements, plan.schema, plan.rows, groups) + obsolete = _materializedarraybytes(WriteRowGroupPlan, 1) + obsolete = _materializedsum(obsolete, + _materializedarraybytes(WriteLeafPlan, length(source.leaves))) + obsolete = _materializedsum(obsolete, + _materializedproduct(2, _MATERIALIZED_OBJECT_BYTES)) + prefixbytes = _materializedproduct(3, + _materializedarraybytes(Int64, plan.rows + 1)) + obsolete = _materializedsum(obsolete, + _materializedproduct(length(source.leaves), prefixbytes)) + _release!(budget, obsolete) + return output + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +struct _WriteDataFrame + bytes::Vector{UInt8} + uncompressed_size::Int64 + first_row_index::Int64 +end + +struct _WriteChunkDictionary{V} + values::V + indices::Vector{UInt64} + payload::Vector{UInt8} + bitwidth::Int +end + +function _writepageestimate(leaf::WriteLeafPlan, firstrow::Int, lastrow::Int) + entries = leaf.entry_offsets[lastrow + 1] - + leaf.entry_offsets[firstrow + 1] + dense = leaf.dense_offsets[lastrow + 1] - + leaf.dense_offsets[firstrow + 1] + levels = Int64(0) + !iszero(leaf.column.max_repetition_level) && + (levels = Base.checked_add(levels, Base.checked_mul(entries, Int64(8)))) + !iszero(leaf.column.max_definition_level) && + (levels = Base.checked_add(levels, Base.checked_mul(entries, Int64(8)))) + raw = if leaf.column.physical == Metadata.Type.BOOLEAN + cld(dense, Int64(8)) + else + leaf.payload_offsets[lastrow + 1] - + leaf.payload_offsets[firstrow + 1] + end + return Base.checked_add(levels, raw) +end + +function _writecandidateboundaries(leaf::WriteLeafPlan, + pagesize::Union{Nothing,Int64}) + rows = leaf.column.rows + iszero(rows) && return Tuple{Int,Int}[] + pagesize === nothing && return Tuple{Int,Int}[(0, rows)] + output = Tuple{Int,Int}[] + firstrow = 0 + while firstrow < rows + lastrow = firstrow + while lastrow < rows + candidate = lastrow + 1 + estimate = _writepageestimate(leaf, firstrow, candidate) + lastrow > firstrow && estimate > pagesize && break + lastrow = candidate + end + lastrow > firstrow || throw(AssertionError( + "writer soft page planner did not consume one row")) + push!(output, (firstrow, lastrow)) + firstrow = lastrow + end + return output +end + +function _writepagecolumn(leaf::WriteLeafPlan, firstrow::Int, lastrow::Int) + entrystart = Int(leaf.entry_offsets[firstrow + 1]) + entrystop = Int(leaf.entry_offsets[lastrow + 1]) + densestart = Int(leaf.dense_offsets[firstrow + 1]) + densestop = Int(leaf.dense_offsets[lastrow + 1]) + repetitions = leaf.column.repetitions === nothing ? nothing : + @view leaf.column.repetitions[(entrystart + 1):entrystop] + definitions = leaf.column.definitions === nothing ? nothing : + @view leaf.column.definitions[(entrystart + 1):entrystop] + values = @view leaf.column.values[(densestart + 1):densestop] + source = leaf.column + column = WriteColumn(source.name, values, source.physical, + source.type_length, source.optional, source.logical, source.converted, + source.path, repetitions, definitions, source.max_repetition_level, + source.max_definition_level, lastrow - firstrow, source.schema) + _columnentrycount(column) > 0 || throw(AssertionError( + "a nonempty row slice produced an empty data page")) + repetitions === nothing || iszero(first(repetitions)) || + throw(ArgumentError("writer data page does not begin at a row boundary")) + actualrows = repetitions === nothing ? _columnentrycount(column) : + count(iszero, repetitions) + actualrows == column.rows || throw(ArgumentError( + "writer page row count does not match its repetition stream")) + return column, densestart, densestop +end + +function _writecapacityargument(err::ArgumentError) + message = err.msg + message isa AbstractString || return false + return message == "encoded Parquet values exceed Int32 bytes" || + message == "Parquet page exceeds Int32 bytes" || + message == "compressed Parquet page exceeds Int32 bytes" +end + +function _writepagecapacity(error) + if error isa LimitError && error.resource == :page_bytes + return _WritePageCapacityError(:page_bytes, error.requested, + error.maximum) + end + if error isa ArgumentError && _writecapacityargument(error) + return _WritePageCapacityError(:page_bytes, typemax(Int64), + typemax(Int32)) + end + return nothing +end + +function _writeencodedpage(leaf::WriteLeafPlan, firstrow::Int, lastrow::Int, + encoding::Metadata.Encoding.T, dictionary, + limits::Limits; checksum::Bool, codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, pageversion::Symbol) + column, densestart, densestop = _writepagecolumn(leaf, firstrow, lastrow) + entries = _columnentrycount(column) + entries <= typemax(Int32) || throw(_WritePageCapacityError( + :page_values, Int64(entries), Int64(typemax(Int32)))) + column.rows <= typemax(Int32) || throw(_WritePageCapacityError( + :page_rows, Int64(column.rows), Int64(typemax(Int32)))) + values = if dictionary === nothing + try + _encodedpayload(column, encoding, limits) + catch err + capacity = _writepagecapacity(err) + capacity === nothing && rethrow() + throw(capacity) + end + else + indices = copy(@view dictionary.indices[(densestart + 1):densestop]) + _encodedictionaryindices(indices, dictionary.bitwidth) + end + try + bytes, headerlength, payloadlength = _datapagebytes(column, values, + encoding, pageversion, limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel) + length(bytes) <= typemax(Int32) || throw(_WritePageCapacityError( + :page_frame_bytes, Int64(length(bytes)), Int64(typemax(Int32)))) + uncompressed = Base.checked_add(Int64(headerlength), + Int64(payloadlength)) + return _WriteDataFrame(bytes, uncompressed, Int64(firstrow)) + catch err + capacity = _writepagecapacity(err) + capacity === nothing && rethrow() + throw(capacity) + end +end + +function _writethrowcapacity(error::_WritePageCapacityError) + throw(LimitError(error.resource, error.requested, error.maximum)) +end + +function _writeappendpages!(output::Vector{_WriteDataFrame}, + leaf::WriteLeafPlan, firstrow::Int, lastrow::Int, + encoding::Metadata.Encoding.T, dictionary, limits::Limits; + checksum::Bool, codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, pageversion::Symbol) + frame = try + _writeencodedpage(leaf, firstrow, lastrow, encoding, dictionary, + limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion) + catch err + err isa _WritePageCapacityError || rethrow() + lastrow - firstrow == 1 && _writethrowcapacity(err) + middle = firstrow + fld(lastrow - firstrow, 2) + _writeappendpages!(output, leaf, firstrow, middle, encoding, + dictionary, limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion) + _writeappendpages!(output, leaf, middle, lastrow, encoding, + dictionary, limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion) + return + end + push!(output, frame) + return +end + +function _writedataframes(leaf::WriteLeafPlan, + pagesize::Union{Nothing,Int64}, encoding::Metadata.Encoding.T, + dictionary, limits::Limits; checksum::Bool, + codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, pageversion::Symbol) + frames = _WriteDataFrame[] + for (firstrow, lastrow) in _writecandidateboundaries(leaf, pagesize) + _writeappendpages!(frames, leaf, firstrow, lastrow, encoding, + dictionary, limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion) + end + isempty(frames) && !iszero(leaf.column.rows) && throw(AssertionError( + "a nonempty leaf chunk produced no data pages")) + return frames +end + +function _writechunkencodings(column::WriteColumn, + encoding::Metadata.Encoding.T; dictionary::Bool=false) + dictionary && return Metadata.Encoding.T[Metadata.Encoding.PLAIN, + Metadata.Encoding.RLE, Metadata.Encoding.RLE_DICTIONARY] + output = Metadata.Encoding.T[] + (!iszero(column.max_repetition_level) || + !iszero(column.max_definition_level)) && + encoding != Metadata.Encoding.RLE && + push!(output, Metadata.Encoding.RLE) + push!(output, encoding) + return output +end + +function _writeaggregatepages(frames::Vector{_WriteDataFrame}, + column::WriteColumn, encoding::Metadata.Encoding.T, + pageversion::Symbol; dictionarypage=nothing, + dictionaryuncompressed::Int64=Int64(0), + capturelocations::Bool=true) + total = dictionarypage === nothing ? 0 : length(dictionarypage) + uncompressed = dictionaryuncompressed + for frame in frames + total = Base.checked_add(total, length(frame.bytes)) + uncompressed = _addgroupsize(uncompressed, frame.uncompressed_size) + end + bytes = UInt8[] + sizehint!(bytes, total) + dictionarypage === nothing || append!(bytes, dictionarypage) + dataoffset = Int64(length(bytes)) + locations = Metadata.PageLocation[] + capturelocations && sizehint!(locations, length(frames)) + for frame in frames + relative = Int64(length(bytes)) + append!(bytes, frame.bytes) + capturelocations && push!(locations, Metadata.PageLocation( + offset=relative, + compressed_page_size=Int32(length(frame.bytes)), + first_row_index=frame.first_row_index)) + end + dictionary = dictionarypage !== nothing + encodings = _writechunkencodings(column, encoding; + dictionary=dictionary) + pagecount = length(frames) + pagecount <= typemax(Int32) || throw(LimitError(:container_elements, + Int64(pagecount), Int64(typemax(Int32)))) + stats = Metadata.PageEncodingStats[] + if dictionary + push!(stats, Metadata.PageEncodingStats( + page_type=Metadata.PageType.DICTIONARY_PAGE, + encoding=Metadata.Encoding.PLAIN, count=Int32(1))) + end + push!(stats, Metadata.PageEncodingStats( + page_type=_datapagetype(pageversion), encoding=encoding, + count=Int32(pagecount))) + return ColumnPages(bytes, uncompressed, dataoffset, + dictionary ? Int64(0) : nothing, encodings, stats, locations) +end + +function _writeencodedchunk(leaf::WriteLeafPlan, + pagesize::Union{Nothing,Int64}, encoding::Metadata.Encoding.T, + limits::Limits; checksum::Bool, codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, pageversion::Symbol, + capturelocations::Bool=true) + frames = _writedataframes(leaf, pagesize, encoding, nothing, limits; + checksum=checksum, codec=codec, compressionlevel=compressionlevel, + pageversion=pageversion) + return _writeaggregatepages(frames, leaf.column, encoding, pageversion; + capturelocations=capturelocations) +end + +function _writechunkdictionary(column::WriteColumn, limits::Limits) + values, indices = _dictionaryentries(column) + length(values) <= typemax(Int32) || throw(_WritePageCapacityError( + :page_values, Int64(length(values)), Int64(typemax(Int32)))) + dictionarycolumn = WriteColumn(column.name, values, column.physical, + column.type_length, false, column.logical, column.converted) + payload = try + _plainpayload(dictionarycolumn, limits) + catch err + capacity = _writepagecapacity(err) + capacity === nothing && rethrow() + throw(capacity) + end + bitwidth = _dictionarybitwidth(length(values)) + bitwidth <= 32 || throw(_WritePageCapacityError(:page_values, + Int64(length(values)), Int64(typemax(UInt32)))) + return _WriteChunkDictionary(values, indices, payload, bitwidth) +end + +function _writedictionaryframe(dictionary::_WriteChunkDictionary, + limits::Limits; checksum::Bool, codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}) + header = Metadata.DictionaryPageHeader( + num_values=Int32(length(dictionary.values)), + encoding=Metadata.Encoding.PLAIN, is_sorted=false) + try + bytes, headerlength, payloadlength = _framedpage(dictionary.payload, + Metadata.PageType.DICTIONARY_PAGE, limits; checksum=checksum, + codec=codec, compressionlevel=compressionlevel, + dictionary_header=header) + return bytes, Base.checked_add(Int64(headerlength), + Int64(payloadlength)) + catch err + capacity = _writepagecapacity(err) + capacity === nothing && rethrow() + throw(capacity) + end +end + +function _writedictionarychunk(leaf::WriteLeafPlan, + pagesize::Union{Nothing,Int64}, limits::Limits; checksum::Bool, + codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, pageversion::Symbol, + capturelocations::Bool=true) + dictionary = _writechunkdictionary(leaf.column, limits) + dictionarypage, dictionaryuncompressed = _writedictionaryframe(dictionary, + limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel) + frames = _writedataframes(leaf, pagesize, + Metadata.Encoding.RLE_DICTIONARY, dictionary, limits; + checksum=checksum, codec=codec, compressionlevel=compressionlevel, + pageversion=pageversion) + return _writeaggregatepages(frames, leaf.column, + Metadata.Encoding.RLE_DICTIONARY, pageversion; + dictionarypage=dictionarypage, + dictionaryuncompressed=dictionaryuncompressed, + capturelocations=capturelocations) +end + +function _writeadaptivecapacity(error) + error isa _WritePageCapacityError && return true + error isa LimitError || return false + return error.resource in (:page_bytes, :page_values, :page_rows, + :page_frame_bytes) +end + +function _writethrowadaptivecapacity(error) + error isa _WritePageCapacityError && _writethrowcapacity(error) + throw(error) +end + +function _writesplitcolumnpages(leaf::WriteLeafPlan, limits::Limits; + pagesize::Union{Nothing,Int64}, checksum::Bool, dictionary::Bool, + codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, pageversion::Symbol, + encoding::Union{Nothing,Metadata.Encoding.T}=nothing, + capturelocations::Bool=true) + encoding === nothing || return _writeencodedchunk(leaf, pagesize, + encoding, limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion, + capturelocations=capturelocations) + dictionary || return _writeencodedchunk(leaf, pagesize, + Metadata.Encoding.PLAIN, limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion, + capturelocations=capturelocations) + leaf.column.physical == Metadata.Type.BOOLEAN && + return _writeencodedchunk(leaf, pagesize, Metadata.Encoding.PLAIN, + limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion, + capturelocations=capturelocations) + plain = nothing + plainerror = nothing + try + plain = _writeencodedchunk(leaf, pagesize, Metadata.Encoding.PLAIN, + limits; checksum=checksum, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion, + capturelocations=capturelocations) + catch err + _writeadaptivecapacity(err) || rethrow() + plainerror = err + end + encoded = nothing + encodederror = nothing + encoded = try + _writedictionarychunk(leaf, pagesize, limits; checksum=checksum, + codec=codec, compressionlevel=compressionlevel, + pageversion=pageversion, + capturelocations=capturelocations) + catch err + _writeadaptivecapacity(err) || rethrow() + encodederror = err + nothing + end + plain === nothing && encoded === nothing && + _writethrowadaptivecapacity(something(plainerror, encodederror)) + plain === nothing && return encoded + encoded === nothing && return plain + length(encoded.bytes) < length(plain.bytes) || return plain + return encoded +end + +function _budgetedsplitcolumnpages(leaf::WriteLeafPlan, limits::Limits, + budget::_LiveByteBudget; pagesize::Union{Nothing,Int64}, + checksum::Bool, dictionary::Bool, + codec::Metadata.CompressionCodec.T, + compressionlevel::Union{Nothing,Integer}, pageversion::Symbol, + encoding::Union{Nothing,Metadata.Encoding.T}=nothing, + capturelocations::Bool=true) + working = _writerpageworkingbytes(leaf.column, dictionary) + _reserve!(budget, working) + pages = try + _writesplitcolumnpages(leaf, limits; pagesize=pagesize, + checksum=checksum, dictionary=dictionary, codec=codec, + compressionlevel=compressionlevel, pageversion=pageversion, + encoding=encoding, capturelocations=capturelocations) + catch + _release!(budget, working) + rethrow() + end + live = _columnpageslivebytes(pages) + live <= working || begin + _release!(budget, working) + throw(AssertionError( + "writer split-page allocation exceeded its materialization preflight")) + end + _release!(budget, working - live) + return pages, live +end diff --git a/src/write_statistics.jl b/src/write_statistics.jl new file mode 100644 index 0000000..f752b6c --- /dev/null +++ b/src/write_statistics.jl @@ -0,0 +1,208 @@ +function _writecolumnorder(element::Metadata.SchemaElement) + semantics = _leafstatisticsemantics(element) + if semantics.floating + return Metadata.ColumnOrder( + IEEE_754_TOTAL_ORDER=Metadata.IEEE754TotalOrder()) + end + return Metadata.ColumnOrder(TYPE_ORDER=Metadata.TypeDefinedOrder()) +end + +function _writecolumnorders(schema::Schema, budget::_LiveByteBudget) + start = _budgetused(budget) + try + count = length(schema.leaves) + _reservearray!(budget, Metadata.ColumnOrder, count) + _reserveobjects!(budget, 2 * count) + orders = Metadata.ColumnOrder[] + sizehint!(orders, count) + for leaf in schema.leaves + push!(orders, _writecolumnorder(leaf.element)) + end + return orders + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end + +function _writeunsignedlittle!(output::Vector{UInt8}, value::T) where + {T<:Unsigned} + length(output) == sizeof(T) || throw(AssertionError( + "writer statistics scratch has the wrong width")) + @inbounds for index in eachindex(output) + output[index] = UInt8(value & T(0xff)) + value >>= 8 + end + return output +end + +function _writefixedstatistic!(output::Vector{UInt8}, + element::Metadata.SchemaElement, value) + physical = element.type_ + if physical == Metadata.Type.BOOLEAN + length(output) == 1 || throw(AssertionError( + "BOOLEAN statistics scratch has the wrong width")) + value isa Bool || throw(ArgumentError( + "BOOLEAN writer statistics value is not Bool")) + output[1] = value ? 0x01 : 0x00 + elseif physical == Metadata.Type.INT32 + value isa Int32 || throw(ArgumentError( + "INT32 writer statistics value is not Int32")) + _writeunsignedlittle!(output, reinterpret(UInt32, value)) + elseif physical == Metadata.Type.INT64 + value isa Int64 || throw(ArgumentError( + "INT64 writer statistics value is not Int64")) + _writeunsignedlittle!(output, reinterpret(UInt64, value)) + elseif physical == Metadata.Type.FLOAT + value isa Float32 || throw(ArgumentError( + "FLOAT writer statistics value is not Float32")) + _writeunsignedlittle!(output, reinterpret(UInt32, value)) + elseif physical == Metadata.Type.DOUBLE + value isa Float64 || throw(ArgumentError( + "DOUBLE writer statistics value is not Float64")) + _writeunsignedlittle!(output, reinterpret(UInt64, value)) + else + length(value) == length(output) || throw(ArgumentError( + "fixed writer statistics value has the wrong width")) + @inbounds for index in eachindex(output) + output[index] = value[index] + end + end + return output +end + +function _writevariablestatistic(value) + value isa AbstractString && return codeunits(value) + value isa AbstractVector{UInt8} && return value + throw(ArgumentError( + "BYTE_ARRAY writer statistics value is not a byte sequence")) +end + +function _writefloatbits(element::Metadata.SchemaElement, value) + physical = element.type_ + physical == Metadata.Type.FLOAT && return reinterpret(UInt32, value::Float32) + physical == Metadata.Type.DOUBLE && return reinterpret(UInt64, value::Float64) + length(value) == 2 || throw(ArgumentError( + "FLOAT16 writer statistics value has the wrong width")) + return UInt16(value[1]) | (UInt16(value[2]) << 8) +end + +function _writecountnans(element::Metadata.SchemaElement, values) + count = Int64(0) + for value in values + _statisticisnan(_writefloatbits(element, value)) && + (count = Base.checked_add(count, Int64(1))) + end + return count +end + +function _writestatisticsmetadata(nulls::Int64, nans::Union{Nothing,Int64}, + lower::Union{Nothing,Vector{UInt8}}, upper::Union{Nothing,Vector{UInt8}}, + budget::_LiveByteBudget) + _reserveobjects!(budget) + exact = lower === nothing ? nothing : true + return Metadata.Statistics( + null_count=nulls, + min_value=lower, + max_value=upper, + is_min_value_exact=exact, + is_max_value_exact=exact, + nan_count=nans, + ) +end + +function _writefixedstatistics(element::Metadata.SchemaElement, values, + comparison::Symbol, nulls::Int64, nans::Union{Nothing,Int64}, + width::Int, limit::Int64, budget::_LiveByteBudget) + isempty(values) && return _writestatisticsmetadata( + nulls, nans, nothing, nothing, budget) + width > limit && return _writestatisticsmetadata( + nulls, nans, nothing, nothing, budget) + _reservearray!(budget, UInt8, width) + _reservearray!(budget, UInt8, width) + scratchcharge = _reservearray!(budget, UInt8, width) + lower = Vector{UInt8}(undef, width) + upper = Vector{UInt8}(undef, width) + scratch = Vector{UInt8}(undef, width) + skipnans = nans !== nothing && nans < length(values) + initialized = false + for value in values + skipnans && _statisticisnan(_writefloatbits(element, value)) && continue + _writefixedstatistic!(scratch, element, value) + if !initialized + copyto!(lower, scratch) + copyto!(upper, scratch) + initialized = true + continue + end + lowerorder = _comparestatisticvalues(element, scratch, lower, comparison) + upperorder = _comparestatisticvalues(element, scratch, upper, comparison) + lowerorder === nothing && throw(AssertionError( + "writer statistics comparator rejected a defined order")) + upperorder === nothing && throw(AssertionError( + "writer statistics comparator rejected a defined order")) + lowerorder < 0 && copyto!(lower, scratch) + upperorder > 0 && copyto!(upper, scratch) + end + initialized || throw(AssertionError( + "writer statistics found no value for a defined order")) + _release!(budget, scratchcharge) + return _writestatisticsmetadata(nulls, nans, lower, upper, budget) +end + +function _writevariablestatistics(element::Metadata.SchemaElement, values, + comparison::Symbol, nulls::Int64, limit::Int64, + budget::_LiveByteBudget) + isempty(values) && return _writestatisticsmetadata( + nulls, nothing, nothing, nothing, budget) + lowerindex = firstindex(values) + upperindex = lowerindex + for index in Iterators.drop(eachindex(values), 1) + raw = _writevariablestatistic(values[index]) + lower = _writevariablestatistic(values[lowerindex]) + upper = _writevariablestatistic(values[upperindex]) + _comparestatisticvalues(element, raw, lower, comparison) < 0 && + (lowerindex = index) + _comparestatisticvalues(element, raw, upper, comparison) > 0 && + (upperindex = index) + end + lowerraw = _writevariablestatistic(values[lowerindex]) + upperraw = _writevariablestatistic(values[upperindex]) + (length(lowerraw) > limit || length(upperraw) > limit) && + return _writestatisticsmetadata( + nulls, nothing, nothing, nothing, budget) + _reservearray!(budget, UInt8, length(lowerraw)) + _reservearray!(budget, UInt8, length(upperraw)) + lower = collect(lowerraw) + upper = collect(upperraw) + return _writestatisticsmetadata(nulls, nothing, lower, upper, budget) +end + +function _writecolumnstatistics(leaf::WriteLeafPlan, + element::Metadata.SchemaElement, limit::Int64, + budget::_LiveByteBudget) + start = _budgetused(budget) + try + entries = Int64(_columnentrycount(leaf.column)) + dense = Int64(length(leaf.column.values)) + dense <= entries || throw(AssertionError( + "writer dense statistics count exceeds its entry count")) + nulls = entries - dense + semantics = _leafstatisticsemantics(element) + nans = semantics.floating ? + _writecountnans(element, leaf.column.values) : nothing + comparison = semantics.floating ? :ieee_total_order : semantics.comparison + comparison === :undefined && return _writestatisticsmetadata( + nulls, nans, nothing, nothing, budget) + width = _statisticplainwidth(element) + width === nothing && return _writevariablestatistics(element, + leaf.column.values, comparison, nulls, limit, budget) + return _writefixedstatistics(element, leaf.column.values, comparison, + nulls, nans, width, limit, budget) + catch + used = _budgetused(budget) + used > start && _release!(budget, used - start) + rethrow() + end +end diff --git a/src/writer.jl b/src/writer.jl deleted file mode 100644 index d8fecb9..0000000 --- a/src/writer.jl +++ /dev/null @@ -1,610 +0,0 @@ -using Tables -using DataAPI -using Thrift -using Snappy -using CodecZstd: ZstdCompressor -using CodecZlib: GzipCompressor -#using CodecLz4: LZ4HCCompressor # wating for CodecLz4.jl devs to fix a bug -using LittleEndianBase128 -using Base.Iterators: partition -using CategoricalArrays: CategoricalArray, CategoricalValue - -using Base: SkipMissing - -if VERSION < v"1.3" - using Missings: nonmissingtype -end - -# a mapping of Julia types to _Type codes in Parquet format -const COL_TYPE_CODE = Dict{DataType, Int32}( - Bool => PAR2._Type.BOOLEAN, - Int32 => PAR2._Type.INT32, - Int64 => PAR2._Type.INT64, - #INT96 => 3, // deprecated, only used by legacy implementations. # not supported by Parquet.jl - Float32 => PAR2._Type.FLOAT, - Float64 => PAR2._Type.DOUBLE, - String => PAR2._Type.BYTE_ARRAY, # BYTE_ARRAY - # FIXED_LEN_BYTE_ARRAY => 7, # current there is no Julia type that we support that maps to this type - ) - -function write_thrift(fileio, thrift_obj) - """write thrift definition to file""" - pos_before_write = position(fileio) - p = TCompactProtocol(TFileTransport(fileio)) - Thrift.write(p, thrift_obj) - pos_after_write = position(fileio) - - size_of_written = pos_after_write - pos_before_write - - size_of_written -end - -function compress_using_codec(colvals::AbstractArray, codec::Integer)::Vector{UInt8} - """Compress `isbits` column types using codec""" - uncompressed_byte_data = reinterpret(UInt8, colvals) |> collect - - if codec == PAR2.CompressionCodec.UNCOMPRESSED - return uncompressed_byte_data - elseif codec == PAR2.CompressionCodec.SNAPPY - compressed_data = Snappy.compress(uncompressed_byte_data) - elseif codec == PAR2.CompressionCodec.GZIP - compressed_data = transcode(GzipCompressor, uncompressed_byte_data) - elseif codec == PAR2.CompressionCodec.LZ4 - error("lz4 is not supported as data compressed with https://github.com/JuliaIO/CodecLz4.jl can't seem to be read by R or Python. If you know how to fix it please help out.") - #compressed_data = transcode(LZ4HCCompressor, uncompressed_byte_data) - elseif codec == PAR2.CompressionCodec.ZSTD - compressed_data = transcode(ZstdCompressor, uncompressed_byte_data) - else - error("not yet implemented") - end - - return compressed_data -end - -function compress_using_codec(colvals::AbstractVector{String}, codec::Int)::Vector{UInt8} - """Compress `String` column using codec""" - # the output - io = IOBuffer() - - # write the values - for val in colvals - # for string it needs to be stored as BYTE_ARRAY which needs the length - # to be the first 4 bytes UInt32 - write(io, val |> sizeof |> UInt32 |> htol) - # write each of the strings one after another - write(io, val) - end - - uncompressed_bytes = take!(io) - return compress_using_codec(uncompressed_bytes, codec) -end - -function write_defn_levels(data_to_compress_io, colvals::AbstractVector{Union{Missing, T}}) where T - """ A function to write definition levels for `Union{Missing, T}`""" - # if there is missing - # use the bit packing algorithm to write the - # definition_levels - bytes_needed = ceil(Int, length(colvals) / 8sizeof(UInt8)) - tmp = UInt32((UInt32(bytes_needed) << 1) | 1) - bitpacking_header = LittleEndianBase128.encode(tmp) - - tmpio = IOBuffer() - not_missing_bits::BitArray = .!ismissing.(colvals) - write(tmpio, not_missing_bits) - seek(tmpio, 0) - - encoded_defn_data = read(tmpio, bytes_needed) - - encoded_defn_data_length = length(bitpacking_header) + bytes_needed - # write the definition data - write(data_to_compress_io, UInt32(encoded_defn_data_length) |> htol) - write(data_to_compress_io, bitpacking_header) - write(data_to_compress_io, encoded_defn_data) -end - -function write_defn_levels(data_to_compress_io, colvals::AbstractVector) - """ A function to write definition levels for NON-missing data - """ - # if there is no missing can just use RLE of one - # using rle - rle_header = LittleEndianBase128.encode(UInt32(length(colvals)) << 1) - repeated_value = UInt8(1) - encoded_defn_data_length = sizeof(rle_header) + sizeof(repeated_value) - - # write the definition data - write(data_to_compress_io, UInt32(encoded_defn_data_length) |> htol) - write(data_to_compress_io, rle_header) - write(data_to_compress_io, repeated_value) -end - -# TODO turn this on when writing dictionary is necessary -# function write_col_dict(fileio, colvals::AbstractArray{T}, codec) where T -# """ write the column dictionary page """ -# # note: `level`s does not return `missing` as a level -# uvals = DataAPI.levels(colvals) -# -# # do not support dictionary with more than 127 levels -# # TODO relax this 127 restriction -# if length(uvals) > 127 -# @warn "More than 127 levels in dictionary. Parquet.jl does not support this at this stage." -# return (offset = missing, uncompressed_size = 0, compressed_size = 0) -# end -# -# if nonmissingtype(T) == String -# # the raw bytes of made of on UInt32 to indicate string length -# # and the content of the string -# # so the formula for dict size is as below -# uncompressed_dict_size = sizeof(UInt32)*length(uvals) + sum(sizeof, uvals) -# else -# uncompressed_dict_size = length(uvals)*sizeof(eltype(uvals)) -# end -# -# compressed_uvals::Vector{UInt8} = compress_using_codec(uvals, codec) -# compressed_dict_size = length(compressed_uvals) -# -# # TODO do the CRC properly -# crc = 0 -# -# # construct dictionary metadata -# dict_page_header = PAR2.PageHeader() -# -# dict_page_header._type = PAR2.PageType.DICTIONARY_PAGE -# dict_page_header.uncompressed_page_size = uncompressed_dict_size -# dict_page_header.compressed_page_size = compressed_dict_size -# dict_page_header.crc = crc -# -# dict_page_header.dictionary_page_header = PAR2.DictionaryPageHeader() -# dict_page_header.dictionary_page_header.num_values = Int32(length(uvals)) -# dict_page_header.dictionary_page_header.encoding = PAR2.Encoding.PLAIN_DICTIONARY -# dict_page_header.dictionary_page_header.is_sorted = false -# -# before_write_page_header_pos = position(fileio) -# -# dict_page_header_size = write_thrift(fileio, dict_page_header) -# -# # write the dictionary data -# write(fileio, compressed_uvals) -# -# return (offset = before_write_page_header_pos, uncompressed_size = uncompressed_dict_size + dict_page_header_size, compressed_size = compressed_dict_size + dict_page_header_size) -# end - - -write_encoded_data(data_to_compress_io, colvals::AbstractVector{Union{Missing, T}}) where T = - write_encoded_data(data_to_compress_io, skipmissing(colvals)) - -function write_encoded_data(data_to_compress_io, colvals::Union{AbstractVector{String}, SkipMissing{S}}) where S <: AbstractVector{Union{Missing, String}} - """ Write encoded data for String type """ - # write the values - for val in colvals - # for string it needs to be stored as BYTE_ARRAY which needs the length - # to be the first 4 bytes UInt32 - write(data_to_compress_io, val |> sizeof |> UInt32 |> htol) - # write each of the strings one after another - write(data_to_compress_io, val) - end -end - -function write_encoded_data(data_to_compress_io, colvals::Union{AbstractVector{Bool}, SkipMissing{S}}) where S <: AbstractVector{Union{Missing, Bool}} - """ Write encoded data for Bool type """ - # write the bitacpked bits - # write a bitarray seems to write 8 bytes at a time - # so write to a tmpio first - no_missing_bit_vec = BitArray(colvals) - bytes_needed = ceil(Int, length(no_missing_bit_vec) / 8sizeof(UInt8)) - tmpio = IOBuffer() - write(tmpio, no_missing_bit_vec) - seek(tmpio, 0) - packed_bits = read(tmpio, bytes_needed) - write(data_to_compress_io, packed_bits) -end - -function write_encoded_data(data_to_compress_io, colvals::AbstractArray) - """ Efficient write of encoded data for `isbits` types""" - @assert isbitstype(eltype(colvals)) - write(data_to_compress_io, colvals |> htol) -end - -function write_encoded_data(data_to_compress_io, colvals::SkipMissing) - """ Write of encoded data for skipped missing types""" - for val in colvals - write(data_to_compress_io, val |> htol) - end -end - -function write_encoded_data(data_to_compress_io, colvals) - """ Write of encoded data for the most general type. - The only requirement is that colvals has to be iterable - """ - for val in skipmissing(colvals) - write(data_to_compress_io, val |> htol) - end -end - -# TODO set the encoding code into a dictionary -function write_col_page(fileio, colvals::AbstractArray, codec, ::Val{PAR2.Encoding.PLAIN}) - """ - Write a chunk of data into a data page using PLAIN encoding where the values - are written back-to-back in memory and then compressed with the codec. - For `String`s, the values are written with length (UInt32), followed by - content; it is NOT null terminated. - """ - - # generate the data page header - data_page_header = PAR2.PageHeader() - - # set up an IO buffer to write to - data_to_compress_io = IOBuffer() - - # write repetition level data - ## do nothing - ## this seems to be related to nested columns - ## and hence is not needed here as we only supported unnested column write - - # write definition levels - write_defn_levels(data_to_compress_io, colvals) - - # write the encoded data - write_encoded_data(data_to_compress_io, colvals) - - data_to_compress::Vector{UInt8} = take!(data_to_compress_io) - - compressed_data::Vector{UInt8} = compress_using_codec(data_to_compress, codec) - - uncompressed_page_size = length(data_to_compress) - compressed_page_size = length(compressed_data) - - data_page_header._type = PAR2.PageType.DATA_PAGE - data_page_header.uncompressed_page_size = uncompressed_page_size - data_page_header.compressed_page_size = compressed_page_size - - # TODO proper CRC - data_page_header.crc = 0 - - data_page_header.data_page_header = PAR2.DataPageHeader() - data_page_header.data_page_header.num_values = Int32(length(colvals)) - data_page_header.data_page_header.encoding = PAR2.Encoding.PLAIN - data_page_header.data_page_header.definition_level_encoding = PAR2.Encoding.RLE - data_page_header.data_page_header.repetition_level_encoding = PAR2.Encoding.RLE - - position_before_page_header_write = position(fileio) - - size_of_page_header_defn_repn = write_thrift(fileio, data_page_header) - - # write data - write(fileio, compressed_data) - - return ( - offset = position_before_page_header_write, - uncompressed_size = uncompressed_page_size + size_of_page_header_defn_repn, - compressed_size = compressed_page_size + size_of_page_header_defn_repn, - ) -end - -function write_col_page(fileio, colvals::AbstractArray, codec, ::Val{PAR2.Encoding.PLAIN_DICTIONARY}) - """write Dictionary encoding data page""" - error("PLAIN_DICTIONARY encoding not implemented yet") - - # TODO finish the implementation - rle_header = LittleEndianBase128.encode(UInt32(length(colvals)) << 1) - repeated_value = UInt8(1) - - encoded_defn_data_length = sizeof(rle_header) + sizeof(repeated_value) - - ## write the encoded data length - write(fileio, encoded_defn_data_length |> UInt32 |> htol) - - write(fileio, rle_header) - write(fileio, repeated_value) - - position(fileio) - - # write the data - - ## firstly, bit pack it - - # the bitwidth to use - bitwidth = ceil(UInt8, log(2, length(uvals))) - # the max bitwidth is 32 according to documentation - @assert bitwidth <= 32 - # to do that I have to figure out the Dictionary index of it - # build a JuliaDict - val_index_dict = Dict(zip(uvals, 1:length(uvals))) - - bitwidth_mask = UInt32(2^bitwidth-1) - - bytes_needed = ceil(Int, bitwidth*length(colvals) / 8) - - bit_packed_encoded_data = zeros(UInt8, bytes_needed) - upto_byte = 1 - - bits_written = 0 - bitsz = 8sizeof(UInt8) - - for val in colvals - bit_packed_val = UInt32(val_index_dict[val]) & bitwidth_mask - if bitwidth_mask <= bitsz - bits_written - bit_packed_encoded_data[upto_byte] = (bit_packed_encoded_data[upto_byte] << bitwidth_mask) | bit_packed_val - else - # this must mean - # bitwidth_mask > bitsz - bits_written - # if the remaining bits is not enough to write a packed number - 42 - end - end -end - -function write_col_page(fileio, colvals::AbstractArray{T}, codec, encoding) where T - error("Page encoding $encoding is yet not implemented.") -end - -write_col(fileio, colvals::CategoricalArray, args...; kwars...) = begin - throw("Currently CategoricalArrays are not supported.") -end - -function write_col(fileio, colvals::AbstractArray{T}, colname, encoding, codec; nchunks = 1) where T - """Write a column to a file""" - # TODO turn writing dictionary on - # Currently, writing the dictionary page is not turned on for any type. - # Normally, for Boolean data, dictionary is not supported. However for other - # data types, dictionary page CAN be supported. However, since Parquet.jl - # only supports writing PLAIN encoding data, hence there is no need to write - # a dictionary page until other dictionary-based encodings are supported - dict_info = (offset = missing, uncompressed_size = 0, compressed_size = 0) - - num_vals_per_chunk = ceil(Int, length(colvals) / nchunks) - - chunk_info = [write_col_page(fileio, val_chunk, codec, Val(encoding)) for val_chunk in partition(colvals, num_vals_per_chunk)] - - sizes = reduce(chunk_info; init = dict_info) do x, y - ( - uncompressed_size = x.uncompressed_size + y.uncompressed_size, - compressed_size = x.compressed_size + y.compressed_size - ) - end - - # write the column metadata - # can probably write the metadata right after the data chunks - col_meta = PAR2.ColumnMetaData() - - col_meta._type = COL_TYPE_CODE[eltype(colvals) |> nonmissingtype] - # these are all the fields - # TODO collect all the encodings used - if eltype(colvals) == Bool - col_meta.encodings = Int32[0, 3] - else - col_meta.encodings = Int32[2, 0, 3] - end - col_meta.path_in_schema = [colname] - col_meta.codec = codec - col_meta.num_values = length(colvals) - - col_meta.total_uncompressed_size = sizes.uncompressed_size - col_meta.total_compressed_size = sizes.compressed_size - - col_meta.data_page_offset = chunk_info[1].offset - if !ismissing(dict_info.offset) - col_meta.dictionary_page_offset = dict_info.offset - end - - # write the column meta data right after the data - # keep track of the position so it can put into the column chunk - # metadata - col_meta_offset = position(fileio) - write_thrift(fileio, col_meta) - - # Prep metadata for the filemetadata - ## column chunk metadata - col_chunk_meta = PAR2.ColumnChunk() - - col_chunk_meta.file_offset = col_meta_offset - col_chunk_meta.meta_data = col_meta - - return ( - data_page_offset = chunk_info[1].offset, - dictionary_page_offset = dict_info.offset, - col_chunk_meta = col_chunk_meta, - col_meta_offset = col_meta_offset - ) -end - -function create_schema_parent_node(ncols) - """Create the parent node in the schema tree""" - schmea_parent_node = PAR2.SchemaElement() - schmea_parent_node.name = "schema" - schmea_parent_node.num_children = ncols - schmea_parent_node -end - -function create_col_schema(type, colname) - """Create a column node in the schema tree for non-strings""" - schema_node = PAR2.SchemaElement() - # look up type code - schema_node._type = COL_TYPE_CODE[type |> nonmissingtype] - schema_node.repetition_type = 1 - schema_node.name = colname - schema_node.num_children = 0 - - schema_node -end - - -function create_col_schema(type::Type{String}, colname) - """create col schema for string""" - schema_node = PAR2.SchemaElement() - # look up type code - schema_node._type = COL_TYPE_CODE[type] - schema_node.repetition_type = 1 - schema_node.name = colname - schema_node.num_children = 0 - - # for string set converted type to UTF8 - schema_node.converted_type = PAR2.ConvertedType.UTF8 - - logicalType = PAR2.LogicalType() - logicalType.STRING = PAR2.StringType() - - schema_node.logicalType = logicalType - - schema_node -end - - -""" - Write a parquet file from a Tables.jl compatible table e.g DataFrame - -io - A writable IO stream -tbl - A Tables.jl columnaccessible table e.g. a DataFrame -compression_code - Default "SNAPPY". The compression codec. The supported - values are "UNCOMPRESSED", "SNAPPY", "ZSTD", "GZIP" -""" -function write_parquet(io::IO, x; compression_codec = "SNAPPY") - tbl = Tables.Columns(x) - - # check that all types are supported - sch = Tables.schema(tbl) - err_msgs = String[] - for type in sch.types - if type <: CategoricalValue - push!(err_msgs, "CategoricalArrays are not supported at this stage. \n") - elseif !(nonmissingtype(type) <: Union{Int32, Int64, Float32, Float64, Bool, String}) - push!(err_msgs, "Column whose `eltype` is $type is not supported at this stage. \n") - end - end - - err_msgs = unique(err_msgs) - if length(err_msgs) > 0 - throw(reduce(*, err_msgs)) - end - - # set the data page encoding - # currently only PLAIN is supported - # TODO add support for other encodings see - # https://github.com/apache/parquet-format/blob/master/Encodings.md - encoding = Encoding.PLAIN - - # convert a string or symbol compression codec into the numeric code - codec = getproperty(PAR2.CompressionCodec, Symbol(uppercase(string(compression_codec)))) - - # figure out the right number of chunks - # TODO test that it works for all supported table - nrows = Tables.rowcount(tbl) - sample_size = min(100, nrows) - rs = collect(Iterators.take(Tables.namedtupleiterator(tbl), sample_size)) - table_size_bytes = Base.summarysize(rs) / sample_size * nrows - - approx_raw_to_parquet_compression_ratio = 6 - approx_post_compression_size = (table_size_bytes / 2^30) / approx_raw_to_parquet_compression_ratio - - # if size is larger than 64mb and has more than 6 rows - if (approx_post_compression_size > 0.064) & (nrows > 6) - recommended_chunks = ceil(Int, approx_post_compression_size / 6) * 6 - else - recommended_chunks = 1 - end - - colnames = String.(Tables.columnnames(tbl)) - _write_parquet( - io, - tbl, - Tables.columnnames(tbl), - recommended_chunks; - encoding = Dict(col => encoding for col in colnames), - codec = Dict(col => codec for col in colnames) - ) -end - -""" - Write a parquet file from a Tables.jl compatible table e.g DataFrame - -path - The file path -tbl - A Tables.jl columnaccessible table e.g. a DataFrame -compression_code - Default "SNAPPY". The compression codec. The supported - values are "UNCOMPRESSED", "SNAPPY", "ZSTD", "GZIP" -""" -function write_parquet(path, x; compression_codec = "SNAPPY") - open(path, "w") do io - write_parquet(io, x; compression_codec=compression_codec) - end -end - -function _write_parquet(io::IO, itr_vectors, colnames, nchunks; ncols = length(itr_vectors), encoding::Dict{String, Int32}, codec::Dict{String, Int32}) - - """Internal method for writing parquet - - itr_vectors - An iterable of `AbstractVector`s containing the values to be - written - colnames - Column names for each of the vectors - path - The output parquet file path - nchunks - The number of chunks/pages to write for each column - ncols - The number of columns. This is provided as an argument for - the case where the `length(itr_vectors)` is not defined, - e.g. lazy loading of remote resources. - encoding - A dictionary mapping from column names to encoding - codec - A dictionary mapping from column names to compression codec - """ - write(io, "PAR1") - - # the + 1 comes from the fact that schema is a tree and there is an extra - # parent node - schemas = Vector{PAR2.SchemaElement}(undef, ncols + 1) - schemas[1] = create_schema_parent_node(ncols) - col_chunk_metas = Vector{PAR2.ColumnChunk}(undef, ncols) - row_group_file_offset = missing - - # write the columns one by one - # TODO parallelize this - nrows = -1 # initialize it - for (coli, (colname_sym, colvals)) in enumerate(zip(colnames, itr_vectors)) - colname = String(colname_sym) - - col_encoding = encoding[colname] - col_codec = codec[colname] - # write the data including metadata - col_info = write_col(io, colvals, colname, col_encoding, col_codec; nchunks = nchunks) - - # the `row_group_file_offset` keeps track of where the data starts, so - # keep it at the dictonary of the first data - if coli == 1 - nrows = length(colvals) - if ismissing(col_info.dictionary_page_offset) - row_group_file_offset = col_info.data_page_offset - else - row_group_file_offset = col_info.dictionary_page_offset - end - end - - col_chunk_metas[coli] = col_info.col_chunk_meta - - # add the schema - schemas[coli + 1] = create_col_schema(eltype(colvals) |> nonmissingtype, colname) - end - - # now all the data is written we write the filemetadata - # finalise it by writing the filemetadata - filemetadata = PAR2.FileMetaData() - filemetadata.version = 1 - filemetadata.schema = schemas - filemetadata.num_rows = nrows - filemetadata.created_by = "Parquet.jl $(Parquet.PARQUET_JL_VERSION)" - - # create row_groups - # TODO do multiple row_groups - row_group = PAR2.RowGroup() - - row_group.columns = col_chunk_metas - row_group.total_byte_size = Int64(sum(x->x.meta_data.total_compressed_size, col_chunk_metas)) - row_group.num_rows = nrows - if ismissing(row_group_file_offset) - error("row_group_file_offset is not set") - else - row_group.file_offset = row_group_file_offset - end - row_group.total_compressed_size = Int64(sum(x->x.meta_data.total_compressed_size, col_chunk_metas)) - - filemetadata.row_groups = [row_group] - - filemetadata_size = write_thrift(io, filemetadata) - - write(io, UInt32(filemetadata_size) |> htol) - write(io, "PAR1") -end diff --git a/test/api.jl b/test/api.jl new file mode 100644 index 0000000..1256d02 --- /dev/null +++ b/test/api.jl @@ -0,0 +1,22 @@ +@testset "public surface" begin + @test !Base.isexported(Parquet, :File) + @test !Base.isexported(Parquet, :Limits) + @test !Base.isexported(Parquet, :Table) + @test !Base.isexported(Parquet, :close!) + @test !Base.isexported(Parquet, :write) + for name in (:BSONValue, :Decimal, :Interval, :JSONValue, :LogicalColumn, + :Timestamp) + @test !Base.isexported(Parquet, name) + end + if VERSION >= v"1.11" + @test Base.ispublic(Parquet, :File) + @test Base.ispublic(Parquet, :Limits) + @test Base.ispublic(Parquet, :Table) + @test Base.ispublic(Parquet, :close!) + @test Base.ispublic(Parquet, :write) + for name in (:BSONValue, :Decimal, :Interval, :JSONValue, :LogicalColumn, + :Timestamp) + @test Base.ispublic(Parquet, name) + end + end +end diff --git a/test/bss.jl b/test/bss.jl new file mode 100644 index 0000000..ab6410b --- /dev/null +++ b/test/bss.jl @@ -0,0 +1,168 @@ +using Random + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end + +const BSS_CORPUS = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + +function bsscorpus(parts...) + return joinpath(BSS_CORPUS, "data", parts...) +end + +function bssbits(values::AbstractVector{T}) where {T} + return reinterpret(Parquet._splitbits(T), values) +end + +# Uncompressed V1 data page of a column chunk: (bytes after the RLE definition levels, value count, chunk metadata). +function bssfixturepage(path::String, column::Int) + file = Parquet.File(path) + meta = TH.decode(copy(file.footer.bytes), MD.FileMetaData) + close(file) + bytes = read(path) + chunk = meta.row_groups[1].columns[column] + md = chunk.meta_data + r = TH.Reader(bytes, md.data_page_offset + 1, length(bytes)) + header = TH.decode(r, MD.PageHeader) + start = md.data_page_offset + TH.consumed(r) + 1 + compressed = view(bytes, start:(start + header.compressed_page_size - 1)) + page = Parquet.decompress(md.codec, compressed, header.uncompressed_page_size) + levellength = Int(reinterpret(UInt32, page[1:4])[1]) + data = page[(5 + levellength):end] + return data, header.data_page_header.num_values, md, meta.schema[column + 1] +end + +@testset "BYTE_STREAM_SPLIT specification example" begin + raw = UInt8[0xaa, 0xbb, 0xcc, 0xdd, 0x00, 0x11, 0x22, 0x33, 0xa3, 0xb4, 0xc5, 0xd6] + split = UInt8[0xaa, 0x00, 0xa3, 0xbb, 0x11, 0xb4, 0xcc, 0x22, 0xc5, 0xdd, 0x33, 0xd6] + ints = collect(reinterpret(Int32, raw)) + floats = collect(reinterpret(Float32, raw)) + @test Parquet.encode_byte_stream_split(ints) == split + @test Parquet.encode_byte_stream_split(floats) == split + @test Parquet.decode_byte_stream_split(Int32, split, 3) == (ints, 13) + @test bssbits(Parquet.decode_byte_stream_split(Float32, split, 3)[1]) == bssbits(floats) + matrix = reshape(raw, 4, 3) + @test Parquet.encode_byte_stream_split_fixed(matrix) == split + @test Parquet.decode_byte_stream_split_fixed(split, 3, 4) == (matrix, 13) + @test Parquet.decode_byte_stream_split(Int32, vcat(UInt8[0x00], split, UInt8[0xff]), 3; offset=2) == (ints, 14) +end + +@testset "BYTE_STREAM_SPLIT bit patterns" begin + f32 = collect(reinterpret(Float32, UInt32[0x7fc00000, 0x7fc0dead, 0xffc00001, 0x80000000, 0x00000000, 0x00000001, 0x7f800000, 0xff800000, 0x3f800000])) + @test bssbits(Parquet.decode_byte_stream_split(Float32, Parquet.encode_byte_stream_split(f32), 9)[1]) == bssbits(f32) + f64 = collect(reinterpret(Float64, UInt64[0x7ff8000000000000, 0x7ff800deadbeef00, 0xfff8000000000001, 0x8000000000000000, 0x0000000000000000, 0x0000000000000001, 0x7ff0000000000000, 0xfff0000000000000])) + @test bssbits(Parquet.decode_byte_stream_split(Float64, Parquet.encode_byte_stream_split(f64), 8)[1]) == bssbits(f64) + for values in (Int32[typemin(Int32), -1, 0, 1, typemax(Int32)], Int64[typemin(Int64), -1, 0, 1, typemax(Int64)]) + @test Parquet.decode_byte_stream_split(eltype(values), Parquet.encode_byte_stream_split(values), 5)[1] == values + end + @test Parquet.encode_byte_stream_split(Float64[]) == UInt8[] + @test Parquet.decode_byte_stream_split(Float64, UInt8[], 0) == (Float64[], 1) + @test Parquet.decode_byte_stream_split(Int64, Parquet.encode_byte_stream_split(Int64[-2]), 1) == (Int64[-2], 9) + @test_throws Parquet.FormatError Parquet.decode_byte_stream_split_fixed(UInt8[], 4, 0) + @test_throws ArgumentError Parquet.encode_byte_stream_split_fixed(Matrix{UInt8}(undef, 0, 4)) + @test Parquet.decode_byte_stream_split_fixed(UInt8[], 0, 3) == (Matrix{UInt8}(undef, 3, 0), 1) + @test Parquet.encode_byte_stream_split_fixed(Matrix{UInt8}(undef, 3, 0)) == UInt8[] + half = reshape(UInt8[0x00, 0x3c, 0x00, 0xbc, 0x00, 0x7e, 0x01, 0x00], 2, 4) + @test Parquet.encode_byte_stream_split_fixed(half) == UInt8[0x00, 0x00, 0x00, 0x01, 0x3c, 0xbc, 0x7e, 0x00] + @test Parquet.decode_byte_stream_split_fixed(Parquet.encode_byte_stream_split_fixed(half), 4, 2)[1] == half +end + +@testset "BYTE_STREAM_SPLIT randomized round trips" begin + rng = MersenneTwister(4242) + for T in (Int32, Int64, Float32, Float64), count in (1, 2, 3, 17, 256, 1001) + values = T <: AbstractFloat ? collect(reinterpret(T, rand(rng, Parquet._splitbits(T), count))) : rand(rng, T, count) + encoded = Parquet.encode_byte_stream_split(values) + @test length(encoded) == count * sizeof(T) + decoded, next = Parquet.decode_byte_stream_split(T, encoded, count) + @test bssbits(decoded) == bssbits(values) && next == length(encoded) + 1 + padded = vcat(rand(rng, UInt8, 3), encoded, rand(rng, UInt8, 2)) + @test Parquet.decode_byte_stream_split(T, padded, count; offset=4)[2] == 4 + length(encoded) + @test bssbits(Parquet.decode_byte_stream_split(T, view(padded, 4:(3 + length(encoded))), count)[1]) == bssbits(values) + slice = Parquet.readrange(Parquet.source(padded), 3, length(encoded)) + @test bssbits(Parquet.decode_byte_stream_split(T, slice, count)[1]) == bssbits(values) + output = Vector{T}(undef, count) + @test Parquet.decode_byte_stream_split!(output, padded; offset=4) == 4 + length(encoded) + @test bssbits(output) == bssbits(values) + end + for width in (1, 2, 5, 16), count in (1, 7, 300) + matrix = rand(rng, UInt8, width, count) + encoded = Parquet.encode_byte_stream_split_fixed(matrix) + @test Parquet.decode_byte_stream_split_fixed(encoded, count, width) == (matrix, length(encoded) + 1) + slice = Parquet.readrange(Parquet.source(encoded), 0, length(encoded)) + @test Parquet.decode_byte_stream_split_fixed(slice, count, width)[1] == matrix + end +end + +@testset "BYTE_STREAM_SPLIT malformed input and limits" begin + F = Parquet.FormatError + L = Parquet.LimitError + encoded = Parquet.encode_byte_stream_split(Float64.(1:10)) + for n in 0:(length(encoded) - 1) + @test_throws F Parquet.decode_byte_stream_split(Float64, encoded[1:n], 10) + end + @test_throws F Parquet.decode_byte_stream_split!(Vector{Int32}(undef, 4), UInt8[0x01, 0x02, 0x03]) + @test_throws F Parquet.decode_byte_stream_split(Float64, encoded, 10; offset=2) + @test_throws ArgumentError Parquet.decode_byte_stream_split(Float64, encoded, -1) + @test_throws L Parquet.decode_byte_stream_split(Float64, encoded, 10; limits=Parquet.Limits(max_container_elements=5)) + @test_throws L Parquet.decode_byte_stream_split(Float64, encoded, 10; limits=Parquet.Limits(max_page_bytes=79)) + @test_throws L Parquet.decode_byte_stream_split(Float64, encoded, big(typemax(Int64)) + 1) + @test Parquet.decode_byte_stream_split(Float64, encoded, 10; limits=Parquet.Limits(max_page_bytes=80))[1] == Float64.(1:10) + @test_throws F Parquet.decode_byte_stream_split_fixed(UInt8[], 1, 4) + @test_throws F Parquet.decode_byte_stream_split_fixed(zeros(UInt8, 15), 4, 4) + @test_throws L Parquet.decode_byte_stream_split_fixed(zeros(UInt8, 16), 4, 4; limits=Parquet.Limits(max_page_bytes=8)) + @test_throws L Parquet.decode_byte_stream_split_fixed(zeros(UInt8, 16), 4, 4; limits=Parquet.Limits(max_string_bytes=3)) + @test_throws L Parquet.decode_byte_stream_split_fixed(zeros(UInt8, 16), 4, 4; limits=Parquet.Limits(max_container_elements=3)) + @test_throws F Parquet.decode_byte_stream_split_fixed(zeros(UInt8, 16), 4, -1) + @test_throws ArgumentError Parquet.decode_byte_stream_split_fixed(zeros(UInt8, 16), -4, 1) + unbounded = Parquet.Limits(max_container_elements=typemax(Int64), max_string_bytes=typemax(Int64), max_page_bytes=typemax(Int64)) + @test_throws F Parquet.decode_byte_stream_split_fixed(UInt8[], typemax(Int) ÷ 2, 4; limits=unbounded) + @test_throws F Parquet.decode_byte_stream_split_fixed(UInt8[], 4, typemax(Int) ÷ 2; limits=unbounded) + @test_throws L Parquet.decode_byte_stream_split_fixed(UInt8[], 0, big(typemax(Int64)) + 1) +end + +@testset "BYTE_STREAM_SPLIT gzip corpus fixture" begin + if !isdir(bsscorpus()) + @info "parquet-testing corpus is not available; skipping byte_stream_split_extended.gzip.parquet" + else + path = bsscorpus("byte_stream_split_extended.gzip.parquet") + pairs = ((1, 2, 2), (3, 4, Float32), (5, 6, Float64), (7, 8, Int32), (9, 10, Int64), (11, 12, 5), (13, 14, 4)) + for (plaincolumn, splitcolumn, kind) in pairs + plaindata, count, plainmd, plainelement = bssfixturepage(path, plaincolumn) + splitdata, splitcount, splitmd, splitelement = bssfixturepage(path, splitcolumn) + @test count == splitcount == 200 && plainmd.statistics.null_count == 0 == splitmd.statistics.null_count + @test splitmd.encodings == [MD.Encoding.RLE, MD.Encoding.BYTE_STREAM_SPLIT] + if kind isa Integer + @test plainelement.type_length == splitelement.type_length == kind + plainvalues, _ = Parquet.decode_plain_fixed(plaindata, count, kind) + splitvalues, next = Parquet.decode_byte_stream_split_fixed(splitdata, count, kind) + @test splitvalues == plainvalues && next == length(splitdata) + 1 + @test Parquet.encode_byte_stream_split_fixed(splitvalues) == splitdata + else + plainvalues, _ = Parquet.decode_plain(kind, plaindata, count) + splitvalues, next = Parquet.decode_byte_stream_split(kind, splitdata, count) + @test bssbits(splitvalues) == bssbits(plainvalues) && next == length(splitdata) + 1 + @test Parquet.encode_byte_stream_split(splitvalues) == splitdata + end + end + end +end + +@testset "BYTE_STREAM_SPLIT zstd corpus fixture" begin + if !isdir(bsscorpus()) + @info "parquet-testing corpus is not available; skipping byte_stream_split.zstd.parquet" + else + path = bsscorpus("byte_stream_split.zstd.parquet") + for (column, T) in ((1, Float32), (2, Float64)) + data, count, md, _ = bssfixturepage(path, column) + @test count == 300 && md.statistics.null_count == 0 + values, next = Parquet.decode_byte_stream_split(T, data, count) + @test next == length(data) + 1 && all(isfinite, values) + @test minimum(values) == reinterpret(T, md.statistics.min_value)[1] + @test maximum(values) == reinterpret(T, md.statistics.max_value)[1] + @test Parquet.encode_byte_stream_split(values) == data + end + end +end diff --git a/test/checksum.jl b/test/checksum.jl new file mode 100644 index 0000000..1e6b4ea --- /dev/null +++ b/test/checksum.jl @@ -0,0 +1,25 @@ +@testset "page CRC32" begin + bytes = collect(codeunits("123456789")) + expected = UInt32(0xcbf43926) + @test Parquet.pagechecksum(bytes) == expected + Parquet.verifypagechecksum(reinterpret(Int32, expected), bytes) + @test_throws Parquet.FormatError Parquet.verifypagechecksum(Int32(0), bytes) + + src = Parquet.source(bytes) + slice = Parquet.readrange(src, 0, length(bytes)) + scratch = Parquet._pagechecksumscratch(slice) + constrained = Parquet._LiveByteBudget(Parquet.Limits( + max_materialized_bytes=scratch - 1)) + @test_throws Parquet.LimitError Parquet.verifypagechecksum( + reinterpret(Int32, expected), slice; budget=constrained) + @test Parquet._budgetused(constrained) == 0 + sufficient = Parquet._LiveByteBudget(Parquet.Limits( + max_materialized_bytes=scratch)) + Parquet.verifypagechecksum(reinterpret(Int32, expected), slice; + budget=sufficient) + @test Parquet._budgetused(sufficient) == 0 + @test_throws Parquet.FormatError Parquet.verifypagechecksum(Int32(0), + slice; budget=sufficient) + @test Parquet._budgetused(sufficient) == 0 + Parquet.close!(src) +end diff --git a/test/codecs.jl b/test/codecs.jl new file mode 100644 index 0000000..886bbd3 --- /dev/null +++ b/test/codecs.jl @@ -0,0 +1,439 @@ +using Random + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end +if !isdefined(Parquet, :decompress) + Base.include(Parquet, joinpath(@__DIR__, "..", "src", "codecs.jl")) +end + +const CODEC_CORPUS = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) +const CODEC_LARGE_PAGES = get(ENV, "PARQUET_TEST_LARGE_PAGES", "0") == "1" +const CC = MD.CompressionCodec +const WRITABLE_CODECS = (CC.UNCOMPRESSED, CC.SNAPPY, CC.GZIP, CC.BROTLI, CC.ZSTD, CC.LZ4_RAW) +const COMPRESSING_CODECS = (CC.SNAPPY, CC.GZIP, CC.BROTLI, CC.ZSTD, CC.LZ4_RAW) + +struct ShiftedCodecBytes <: AbstractVector{UInt8} + bytes::Vector{UInt8} +end + +function Base.IndexStyle(::Type{ShiftedCodecBytes}) + return IndexLinear() +end + +function Base.size(bytes::ShiftedCodecBytes) + return (length(bytes.bytes),) +end + +function Base.axes(bytes::ShiftedCodecBytes) + return (2:(length(bytes.bytes) + 1),) +end + +function Base.getindex(bytes::ShiftedCodecBytes, index::Int) + checkbounds(bytes, index) + return bytes.bytes[index - 1] +end + +function codeccorpus(parts...) + return joinpath(CODEC_CORPUS, "data", parts...) +end + +# Bytes with mixed entropy: runs, text, and random noise. +function codecsample(rng::AbstractRNG, count::Int) + output = UInt8[] + while length(output) < count + kind = rand(rng, 1:3) + kind == 1 && append!(output, fill(rand(rng, UInt8), rand(rng, 1:64))) + kind == 2 && append!(output, codeunits("parquet page ")) + kind == 3 && append!(output, rand(rng, UInt8, rand(rng, 1:32))) + end + return output[1:count] +end + +function bigendian32(value::Integer) + return reinterpret(UInt8, [hton(UInt32(value))]) +end + +# One Hadoop LZ4 block can contain multiple compressed chunks. +function hadoopblock(chunks::Vector{Vector{UInt8}}) + output = UInt8[] + append!(output, bigendian32(sum(length, chunks))) + for chunk in chunks + block = Parquet.compress(CC.LZ4_RAW, chunk) + append!(output, bigendian32(length(block))) + append!(output, block) + end + return output +end + +function hadoopframe(chunks::Vector{Vector{UInt8}}) + return vcat((hadoopblock([chunk]) for chunk in chunks)...) +end + +# Decompressed pages of one column chunk: (kind, uncompressed level bytes, data bytes, header, metadata). +function codeccorpuspages(path::String, column::Int; limits=Parquet.Limits()) + file = Parquet.File(path) + meta = TH.decode(copy(file.footer.bytes), MD.FileMetaData) + md = meta.row_groups[1].columns[column].meta_data + start = Int64(md.data_page_offset) + dictionary = md.dictionary_page_offset + dictionary !== nothing && dictionary > 0 && (start = min(start, Int64(dictionary))) + stop = start + md.total_compressed_size + pages = Any[] + position = start + while position < stop + frame = Parquet.readpage(file.source, position, stop, limits) + header = frame.header + v2 = header.data_page_header_v2 + if v2 !== nothing + levellength = Int(v2.definition_levels_byte_length + v2.repetition_levels_byte_length) + payload = collect(frame.payload) + encoded = payload[(levellength + 1):end] + expected = header.uncompressed_page_size - levellength + compressed = something(v2.is_compressed, true) + data = isempty(encoded) && expected == 0 ? UInt8[] : + Parquet.decompress(compressed ? md.codec : CC.UNCOMPRESSED, encoded, expected; limits=limits) + push!(pages, (kind=:v2, levels=payload[1:levellength], data=data, header=header, md=md)) + else + kind = header.dictionary_page_header !== nothing ? :dict : :v1 + data = collect(Parquet.decompress(md.codec, frame.payload, header.uncompressed_page_size; limits=limits)) + push!(pages, (kind=kind, levels=UInt8[], data=data, header=header, md=md)) + end + position = Parquet.pageend(frame) + end + close(file) + return pages +end + +# Data section of a V1 page after its length-prefixed RLE definition levels. +function afterlevels(data::Vector{UInt8}) + levellength = Int(reinterpret(UInt32, data[1:4])[1]) + return data[(5 + levellength):end] +end + +@testset "codec table" begin + @test all(Parquet.codecreadable, (CC.UNCOMPRESSED, CC.SNAPPY, CC.GZIP, CC.BROTLI, CC.LZ4, CC.ZSTD, CC.LZ4_RAW)) + @test !Parquet.codecreadable(CC.LZO) && !Parquet.codecreadable(CC.T(42)) + @test all(Parquet.codecwritable, WRITABLE_CODECS) + @test !Parquet.codecwritable(CC.LZ4) && !Parquet.codecwritable(CC.LZO) && !Parquet.codecwritable(CC.T(42)) + @test Parquet.codecname(CC.ZSTD) == "ZSTD" && Parquet.codecname(CC.T(42)) == "CompressionCodec.T(42)" + @test Parquet._readbe32(fill(UInt8(0xff), 4), 1) == Int64(typemax(UInt32)) + # An LZO file is well formed but permanently unsupported, so it is an + # unsupported feature rather than an invalid file. An unknown codec stays a + # format error, because it can equally mean corruption. + @test_throws Parquet.UnsupportedFeatureError Parquet.decompress(CC.LZO, + UInt8[0x00], 1) + @test_throws Parquet.FormatError Parquet.decompress(CC.T(42), UInt8[0x00], 1) + @test_throws ArgumentError Parquet.compress(CC.LZO, UInt8[0x00]) + @test_throws ArgumentError Parquet.compress(CC.LZ4, UInt8[0x00]) + @test_throws ArgumentError Parquet.compress(CC.T(42), UInt8[0x00]) + @test_throws ArgumentError Parquet.compress(CC.UNCOMPRESSED, UInt8[0x00]; level=1) +end + +@testset "codec round trips" begin + rng = MersenneTwister(2026) + samples = [UInt8[], UInt8[0x2a], zeros(UInt8, 255), rand(rng, UInt8, 4096), codecsample(rng, 100_000), + collect(codeunits(repeat("parquet", 5000)))] + for codec in WRITABLE_CODECS, sample in samples + encoded = Parquet.compress(codec, sample) + @test encoded isa Vector{UInt8} + decoded = Parquet.decompress(codec, encoded, length(sample)) + @test decoded == sample && decoded isa Vector{UInt8} + codec == CC.UNCOMPRESSED && @test encoded == sample && encoded !== sample + padded = vcat(UInt8[0xff], encoded, UInt8[0xee]) + @test Parquet.decompress(codec, view(padded, 2:(1 + length(encoded))), length(sample)) == sample + slice = Parquet.readrange(Parquet.source(padded), 1, length(encoded)) + @test Parquet.decompress(codec, slice, length(sample)) == sample + @test Parquet.decompress(codec, Parquet.compress(codec, view(padded, 2:(1 + length(encoded)))), length(encoded)) == encoded + end + sample = codecsample(rng, 20_000) + for (codec, levels) in ((CC.GZIP, (0, 1, 9)), (CC.ZSTD, (-5, 1, 19)), (CC.BROTLI, (0, 5, 11)), (CC.LZ4_RAW, (0, 1, 12))) + for level in levels + @test Parquet.decompress(codec, Parquet.compress(codec, sample; level=level), length(sample)) == sample + end + end + @test_throws ArgumentError Parquet.compress(CC.GZIP, sample; level=10) + @test_throws ArgumentError Parquet.compress(CC.BROTLI, sample; level=12) + @test_throws ArgumentError Parquet.compress(CC.LZ4_RAW, sample; level=-1) + @test_throws ArgumentError Parquet.compress(CC.ZSTD, sample; level=23) + @test_throws ArgumentError Parquet.compress(CC.SNAPPY, sample; level=1) + bytes = UInt8[1, 2, 3] + @test Parquet.decompress(CC.UNCOMPRESSED, bytes, 3) === bytes + @test_throws Parquet.FormatError Parquet.decompress(CC.UNCOMPRESSED, bytes, 2) + @test_throws Parquet.FormatError Parquet.decompress(CC.UNCOMPRESSED, bytes, 4) +end + +@testset "expected size enforcement" begin + rng = MersenneTwister(7) + sample = codecsample(rng, 3000) + for codec in WRITABLE_CODECS + encoded = Parquet.compress(codec, sample) + for wrong in (length(sample) - 1, length(sample) + 1, 0, 1, 2 * length(sample)) + @test_throws Parquet.FormatError Parquet.decompress(codec, encoded, wrong) + end + @test_throws Parquet.FormatError Parquet.decompress(codec, encoded, -1) + @test_throws Parquet.LimitError Parquet.decompress(codec, encoded, length(sample); + limits=Parquet.Limits(max_page_bytes=length(sample) - 1)) + @test_throws Parquet.LimitError Parquet.decompress(codec, encoded, length(sample); + limits=Parquet.Limits(max_page_bytes=min(length(sample), length(encoded)) - 1)) + @test Parquet.decompress(codec, encoded, length(sample); + limits=Parquet.Limits(max_page_bytes=max(length(sample), length(encoded)))) == sample + @test_throws Parquet.LimitError Parquet.decompress(codec, encoded, typemax(Int64)) + @test_throws Parquet.LimitError Parquet.decompress(codec, encoded, Int64(2)^40) + end + for codec in (CC.SNAPPY, CC.GZIP, CC.BROTLI, CC.ZSTD, CC.LZ4_RAW, CC.LZ4) + @test_throws Parquet.FormatError Parquet.decompress(codec, UInt8[], 0) + @test_throws Parquet.FormatError Parquet.decompress(codec, UInt8[], 1) + end + for codec in COMPRESSING_CODECS + empty = Parquet.compress(codec, UInt8[]) + @test !isempty(empty) && Parquet.decompress(codec, empty, 0) == UInt8[] + @test_throws Parquet.FormatError Parquet.decompress(codec, empty, 1) + end + strided = view(UInt8[1, 9, 2, 9, 3], 1:2:5) + @test Parquet.decompress(CC.SNAPPY, Parquet.compress(CC.SNAPPY, strided), 3) == UInt8[1, 2, 3] + + shiftedencoded = Parquet.compress(CC.SNAPPY, sample) + shiftedsrc = Parquet.source(ShiftedCodecBytes(shiftedencoded)) + shiftedslice = Parquet.readrange(shiftedsrc, 0, length(shiftedencoded)) + copycharge = Parquet._contiguouscopycharge(shiftedslice) + constrained = Parquet._LiveByteBudget(Parquet.Limits( + max_materialized_bytes=copycharge - 1)) + @test_throws Parquet.LimitError Parquet.decompress(CC.SNAPPY, + shiftedslice, length(sample); budget=constrained) + @test Parquet._budgetused(constrained) == 0 + sufficient = Parquet._LiveByteBudget(Parquet.Limits( + max_materialized_bytes=copycharge)) + @test Parquet.decompress(CC.SNAPPY, shiftedslice, length(sample); + budget=sufficient) == sample + @test Parquet._budgetused(sufficient) == 0 + Parquet.close!(shiftedsrc) + closedsource = Parquet.source(Parquet.compress(CC.ZSTD, sample)) + slice = Parquet.readrange(closedsource, 0, Parquet.sourcelength(closedsource)) + Parquet.close!(closedsource) + @test_throws ArgumentError Parquet.decompress(CC.ZSTD, slice, length(sample)) +end + +@testset "malformed and truncated streams" begin + rng = MersenneTwister(11) + sample = codecsample(rng, 2000) + for codec in COMPRESSING_CODECS + encoded = Parquet.compress(codec, sample) + for n in 0:(length(encoded) - 1) + @test_throws Parquet.FormatError Parquet.decompress(codec, encoded[1:n], length(sample)) + end + @test_throws Parquet.FormatError Parquet.decompress(codec, vcat(encoded, UInt8[0x00]), length(sample)) + @test_throws Parquet.FormatError Parquet.decompress(codec, vcat(encoded, encoded), length(sample)) + @test_throws Parquet.FormatError Parquet.decompress(codec, rand(rng, UInt8, 64), 64) + @test_throws Parquet.FormatError Parquet.decompress(codec, zeros(UInt8, 64), 64) + @test_throws Parquet.FormatError Parquet.decompress(codec, fill(0xff, 64), 64) + outcomes = Set{Symbol}() + for trial in 1:300 + mutated = copy(encoded) + for _ in 1:rand(rng, 1:3) + mutated[rand(rng, eachindex(mutated))] = rand(rng, UInt8) + end + result = try + Parquet.decompress(codec, mutated, length(sample)) + :ok + catch err + err + end + if result === :ok + push!(outcomes, :ok) + else + @test result isa Union{Parquet.FormatError,Parquet.LimitError} + push!(outcomes, nameof(typeof(result))) + end + end + @test :FormatError in outcomes + end +end + +@testset "concatenated members" begin + rng = MersenneTwister(5) + a = codecsample(rng, 1500) + b = rand(rng, UInt8, 700) + for codec in (CC.GZIP, CC.ZSTD) + joined = vcat(Parquet.compress(codec, a), Parquet.compress(codec, b)) + @test Parquet.decompress(codec, joined, length(a) + length(b)) == vcat(a, b) + @test_throws Parquet.FormatError Parquet.decompress(codec, joined, length(a)) + @test_throws Parquet.FormatError Parquet.decompress(codec, joined, length(a) + length(b) + 1) + triple = vcat(joined, Parquet.compress(codec, UInt8[])) + @test Parquet.decompress(codec, triple, length(a) + length(b)) == vcat(a, b) + end + for codec in (CC.BROTLI, CC.SNAPPY, CC.LZ4_RAW) + joined = vcat(Parquet.compress(codec, a), Parquet.compress(codec, b)) + @test_throws Parquet.FormatError Parquet.decompress(codec, joined, length(a) + length(b)) + end +end + +@testset "deprecated LZ4 framing" begin + rng = MersenneTwister(9) + chunks = [codecsample(rng, 70_000), rand(rng, UInt8, 1000), UInt8[0x01], codecsample(rng, 300)] + whole = vcat(chunks...) + framed = hadoopframe(chunks) + @test Parquet.decompress(CC.LZ4, framed, length(whole)) == whole + @test Parquet.decompress(CC.LZ4, hadoopframe([whole]), length(whole)) == whole + @test Parquet.decompress(CC.LZ4, hadoopblock(chunks), length(whole)) == whole + single = hadoopframe([chunks[2]]) + @test Parquet.decompress(CC.LZ4, single, 1000) == chunks[2] + raw = Parquet.compress(CC.LZ4_RAW, whole) + @test Parquet.decompress(CC.LZ4, raw, length(whole)) == whole + emptypair = zeros(UInt8, 8) + @test Parquet.decompress(CC.LZ4, vcat(emptypair, single), 1000) == chunks[2] + @test Parquet.decompress(CC.LZ4, vcat(single, emptypair), 1000) == chunks[2] + @test Parquet.decompress(CC.LZ4, emptypair, 0) == UInt8[] + @test Parquet.decompress(CC.LZ4, bigendian32(0), 0) == UInt8[] + @test Parquet.decompress(CC.LZ4, + vcat(bigendian32(0), bigendian32(1), UInt8[0x00]), 0) == UInt8[] + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, vcat(single, UInt8[0x00]), 1000) + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, vcat(single, UInt8[0, 0, 0, 1, 0, 0, 0, 0]), 1000) + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, single, 999) + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, single, 1001) + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, single, 0) + for n in 0:(length(single) - 1) + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, single[1:n], 1000) + end + badoriginal = copy(single) + badoriginal[4] = 0xff + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, badoriginal, 1000) + badcompressed = copy(single) + badcompressed[8] ⊻= 0x01 + @test_throws Parquet.FormatError Parquet.decompress(CC.LZ4, badcompressed, 1000) + @test_throws Parquet.LimitError Parquet.decompress(CC.LZ4, framed, length(whole); limits=Parquet.Limits(max_page_bytes=1000)) + outcomes = Set{Symbol}() + for trial in 1:300 + mutated = copy(framed) + for _ in 1:rand(rng, 1:3) + mutated[rand(rng, eachindex(mutated))] = rand(rng, UInt8) + end + result = try + Parquet.decompress(CC.LZ4, mutated, length(whole)) + :ok + catch err + err + end + if result === :ok + push!(outcomes, :ok) + else + @test result isa Union{Parquet.FormatError,Parquet.LimitError} + push!(outcomes, nameof(typeof(result))) + end + end + @test :FormatError in outcomes +end + +@testset "official corpus codec fixtures" begin + if !isdir(codeccorpus()) + @warn "parquet-testing corpus not found; skipping codec corpus tests" CODEC_CORPUS + else + # SNAPPY: the compressed checksum fixture must decompress to its uncompressed twin, page by page + snappy = [codeccorpuspages(codeccorpus("datapage_v1-snappy-compressed-checksum.parquet"), column) for column in 1:2] + plain = [codeccorpuspages(codeccorpus("datapage_v1-uncompressed-checksum.parquet"), column) for column in 1:2] + @test [[page.data for page in column] for column in snappy] == [[page.data for page in column] for column in plain] + @test all(page -> page.kind === :v1 && page.header.crc !== nothing && length(page.data) == 10240, snappy[1]) + a = vcat([Parquet.decode_plain(Int32, page.data, Int(page.header.data_page_header.num_values))[1] for page in snappy[1]]...) + @test length(a) == 5120 && sum(Int64, a) == 43118090240 && a[1:4] == Int32[50462976, 117835012, 185207048, 252579084] + snappytable = Parquet.Table(codeccorpus("datapage_v1-snappy-compressed-checksum.parquet")) + @test length(snappytable) == 5120 + @test sum(Int64, snappytable.columns.a) == 43118090240 + @test sum(Int64, snappytable.columns.b) == 129016125440 + close(snappytable) + types = codeccorpuspages(codeccorpus("alltypes_plain.snappy.parquet"), 1) + @test [(page.kind, length(page.data)) for page in types] == [(:dict, 8), (:v1, 9)] + @test [(page.kind, length(page.data)) for page in codeccorpuspages(codeccorpus("alltypes_plain.snappy.parquet"), 11)] == [(:dict, 24), (:v1, 9)] + v2 = codeccorpuspages(codeccorpus("datapage_v2.snappy.parquet"), 5) + @test [(page.kind, length(page.levels), length(page.data)) for page in v2] == [(:dict, 0, 12), (:v2, 8, 4)] + @test v2[2].header.data_page_header_v2.repetition_levels_byte_length == 3 + emptyv2 = codeccorpuspages(codeccorpus("datapage_v2_empty_datapage.snappy.parquet"), 1) + @test [(page.kind, length(page.levels), length(page.data)) for page in emptyv2] == [(:v2, 2, 0)] + nested = codeccorpuspages(codeccorpus("nested_lists.snappy.parquet"), 1) + @test [(page.kind, length(page.data)) for page in nested] == [(:dict, 30), (:v1, 33)] + # GZIP: concatenated members and V2 pages + gzip = codeccorpuspages(codeccorpus("concatenated_gzip_members.parquet"), 1)[1] + @test gzip.kind === :v2 && length(gzip.levels) == 3 && reinterpret(Int64, gzip.data) == 1:513 + booleans = codeccorpuspages(codeccorpus("rle_boolean_encoding.parquet"), 1)[1] + @test booleans.kind === :v2 && length(booleans.levels) == 13 && length(booleans.data) == 13 + @test booleans.header.data_page_header_v2.repetition_levels_byte_length == 2 && booleans.data[1:4] == UInt8[0x09, 0x00, 0x00, 0x00] + for (plaincolumn, splitcolumn, T) in ((3, 4, Float32), (5, 6, Float64), (7, 8, Int32), (9, 10, Int64)) + plainpage = codeccorpuspages(codeccorpus("byte_stream_split_extended.gzip.parquet"), plaincolumn)[1] + splitpage = codeccorpuspages(codeccorpus("byte_stream_split_extended.gzip.parquet"), splitcolumn)[1] + plainvalues = Parquet.decode_plain(T, afterlevels(plainpage.data), 200)[1] + splitvalues = Parquet.decode_byte_stream_split(T, afterlevels(splitpage.data), 200)[1] + @test reinterpret(Parquet._splitbits(T), splitvalues) == reinterpret(Parquet._splitbits(T), plainvalues) + end + # BROTLI: small pages decode; the 1 GiB dictionary page is rejected by the default limit before allocation + value = codeccorpuspages(codeccorpus("large_string_map.brotli.parquet"), 2) + @test [(page.kind, length(page.data)) for page in value] == [(:dict, 4), (:v1, 15)] + @test reinterpret(Int32, value[1].data) == Int32[1] + @test_throws Parquet.LimitError codeccorpuspages(codeccorpus("large_string_map.brotli.parquet"), 1) + @test_throws Parquet.LimitError Parquet.decompress(CC.BROTLI, UInt8[0x00], 1073741828) + if CODEC_LARGE_PAGES + key = codeccorpuspages(codeccorpus("large_string_map.brotli.parquet"), 1; limits=Parquet.Limits(max_page_bytes=Int64(2)^31)) + @test [(page.kind, length(page.data)) for page in key] == [(:dict, 1073741828), (:v1, 15), (:v1, 1073741840)] + end + # ZSTD + delta = codeccorpuspages(codeccorpus("delta_length_byte_array.parquet"), 1)[1] + @test delta.kind === :v2 && length(delta.levels) == 3 && length(delta.data) == 23711 + @test Parquet.decode_delta_length_byte_array(delta.data, 1000)[1] == [Vector{UInt8}(codeunits("apple_banana_mango$((index - 1)^2)")) for index in 1:1000] + split = codeccorpuspages(codeccorpus("byte_stream_split.zstd.parquet"), 2)[1] + doubles = Parquet.decode_byte_stream_split(Float64, afterlevels(split.data), 300)[1] + @test minimum(doubles) == reinterpret(Float64, split.md.statistics.min_value)[1] + @test maximum(doubles) == reinterpret(Float64, split.md.statistics.max_value)[1] + alp = codeccorpuspages(codeccorpus("alp_extended.zstd.parquet"), 1)[1] + @test alp.kind === :v1 && length(alp.data) == 24583 && length(afterlevels(alp.data)) == 4 * 6144 + emptyzstd = codeccorpuspages(codeccorpus("page_v2_empty_compressed.parquet"), 1) + @test [(page.kind, length(page.levels), length(page.data)) for page in emptyzstd] == [(:dict, 0, 0), (:v2, 2, 1)] + @test emptyzstd[1].header.compressed_page_size == 9 && emptyzstd[1].header.uncompressed_page_size == 0 + @test emptyzstd[2].data == UInt8[0x00] && emptyzstd[2].header.data_page_header_v2.num_nulls == 10 + # LZ4_RAW and the deprecated LZ4 codec decode the same data + raw = codeccorpuspages(codeccorpus("lz4_raw_compressed_larger.parquet"), 1) + hadoop = codeccorpuspages(codeccorpus("hadoop_lz4_compressed_larger.parquet"), 1) + @test length(raw) == 1 && length(raw[1].data) == 400000 && raw[1].data == hadoop[1].data + @test raw[1].md.codec == CC.LZ4_RAW && hadoop[1].md.codec == CC.LZ4 + small = codeccorpuspages(codeccorpus("lz4_raw_compressed.parquet"), 1) + @test [(page.kind, length(page.data)) for page in small] == [(:v1, 32)] + @test reinterpret(Int64, small[1].data) == Int64[1593604800, 1593604800, 1593604801, 1593604801] + @test [length(codeccorpuspages(codeccorpus("lz4_raw_compressed.parquet"), column)[1].data) for column in 2:3] == [28, 38] + for column in 1:3 + framed = codeccorpuspages(codeccorpus("hadoop_lz4_compressed.parquet"), column) + bare = codeccorpuspages(codeccorpus("non_hadoop_lz4_compressed.parquet"), column) + @test [page.kind for page in framed] == [:dict, :v1] == [page.kind for page in bare] + @test framed[1].data == bare[1].data && length(framed[1].data) == framed[1].header.uncompressed_page_size + end + @test reinterpret(Int64, codeccorpuspages(codeccorpus("hadoop_lz4_compressed.parquet"), 1)[1].data) == Int64[1593604800, 1593604801] + for fixture in ("hadoop_lz4_compressed.parquet", "non_hadoop_lz4_compressed.parquet", + "lz4_raw_compressed.parquet") + table = Parquet.Table(codeccorpus(fixture)) + @test table.columns.c0 == Int64[1593604800, 1593604800, 1593604801, 1593604801] + @test table.columns.c1 == Vector{UInt8}[collect(codeunits(value)) for value in ("abc", "def", "abc", "def")] + @test isequal(table.columns.v11, Union{Missing,Float64}[42.0, 7.7, 42.125, 7.7]) + close(table) + end + # seeded mutations of a real SNAPPY page payload + file = Parquet.File(codeccorpus("datapage_v1-snappy-compressed-checksum.parquet")) + meta = TH.decode(copy(file.footer.bytes), MD.FileMetaData) + md = meta.row_groups[1].columns[1].meta_data + frame = Parquet.readpage(file.source, Int64(md.data_page_offset), Int64(md.data_page_offset + md.total_compressed_size), Parquet.Limits()) + payload = collect(frame.payload) + close(file) + rng = MersenneTwister(99) + for trial in 1:300 + mutated = copy(payload) + for _ in 1:rand(rng, 1:2) + mutated[rand(rng, eachindex(mutated))] = rand(rng, UInt8) + end + result = try + Parquet.decompress(CC.SNAPPY, mutated, 10240) + :ok + catch err + err + end + @test result === :ok || result isa Union{Parquet.FormatError,Parquet.LimitError} + end + end +end diff --git a/test/column.jl b/test/column.jl new file mode 100644 index 0000000..280d915 --- /dev/null +++ b/test/column.jl @@ -0,0 +1,1322 @@ +using Random + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end +if !isdefined(Parquet, :readpage) + Base.include(Parquet, joinpath(@__DIR__, "..", "src", "page.jl")) +end +if !isdefined(Parquet, :readcolumn) + Base.include(Parquet, joinpath(@__DIR__, "..", "src", "column.jl")) +end + +const COLUMN_CORPUS = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + +function columncorpus(parts...) + return joinpath(COLUMN_CORPUS, "data", parts...) +end + +function columnleaf(type; repetition=MD.FieldRepetitionType.REQUIRED, width=nothing) + return MD.SchemaElement(name="value", type_=type, repetition_type=repetition, + type_length=width === nothing ? nothing : Int32(width)) +end + +function columnplain(values; width=nothing) + eltype(values) == Vector{UInt8} || return Parquet.encode_plain(values) + width === nothing && return Parquet.encode_plain_byte_array(values) + matrix = isempty(values) ? Matrix{UInt8}(undef, width, 0) : reduce(hcat, values) + return Parquet.encode_plain_fixed(matrix) +end + +function columnlevels(levels, maxlevel::Int) + return Parquet.encode_hybrid(UInt64.(levels), Parquet._levelbitwidth(maxlevel); length_prefix=true) +end + +function columnv1(count::Integer; encoding=MD.Encoding.PLAIN, + levelencoding=MD.Encoding.RLE, repetitionencoding=MD.Encoding.RLE) + return MD.DataPageHeader(num_values=Int32(count), encoding=encoding, definition_level_encoding=levelencoding, + repetition_level_encoding=repetitionencoding) +end + +function columnpage(payload::Vector{UInt8}; type=MD.PageType.DATA_PAGE, v1=nothing, + index=nothing, dict=nothing, v2=nothing, + crc=:valid, compressed=length(payload), uncompressed=compressed) + crcvalue = crc === :valid ? reinterpret(Int32, Parquet.pagechecksum(payload)) : crc === :none ? nothing : Int32(crc) + header = MD.PageHeader(type_=type, uncompressed_page_size=Int32(uncompressed), compressed_page_size=Int32(compressed), + crc=crcvalue, data_page_header=v1, index_page_header=index, + dictionary_page_header=dict, data_page_header_v2=v2) + return vcat(TH.encode(header), payload) +end + +# A V1 data page: optional definition levels then PLAIN values for the present slots. +function datapage(values; levels=nothing, maxlevel=0, width=nothing, extra=UInt8[], crc=:valid, + encoding=MD.Encoding.PLAIN, levelencoding=MD.Encoding.RLE, repetitions=nothing, + maxrepetition=0, repetitionencoding=MD.Encoding.RLE) + count = levels === nothing ? + (repetitions === nothing ? length(values) : length(repetitions)) : length(levels) + repetitions === nothing || length(repetitions) == count || + throw(ArgumentError("repetition count differs from entry count")) + repetition = repetitions === nothing ? UInt8[] : columnlevels(repetitions, maxrepetition) + definition = levels === nothing ? UInt8[] : columnlevels(levels, maxlevel) + payload = vcat(repetition, definition, columnplain(values; width=width), extra) + header = columnv1(count; encoding=encoding, levelencoding=levelencoding, + repetitionencoding=repetitionencoding) + return columnpage(payload; v1=header, crc=crc) +end + +function datapagev2(values; levels=nothing, maxlevel=0, width=nothing, + repetition=UInt8[], definition=nothing, encoding=MD.Encoding.PLAIN, + nulls=nothing, rows=nothing, is_compressed=false, extra=UInt8[], crc=:valid) + valuecount = levels === nothing ? length(values) : length(levels) + definitions = if definition !== nothing + definition + elseif levels === nothing || maxlevel == 0 + UInt8[] + else + Parquet.encode_hybrid(UInt64.(levels), Parquet._levelbitwidth(maxlevel)) + end + data = columnplain(values; width=width) + payload = vcat(repetition, definitions, data, extra) + missingcount = something(nulls, levels === nothing ? 0 : count(level -> level != maxlevel, levels)) + header = MD.DataPageHeaderV2( + num_values=Int32(valuecount), + num_nulls=Int32(missingcount), + num_rows=Int32(something(rows, valuecount)), + encoding=encoding, + definition_levels_byte_length=Int32(length(definitions)), + repetition_levels_byte_length=Int32(length(repetition)), + is_compressed=is_compressed, + ) + return columnpage(payload; type=MD.PageType.DATA_PAGE_V2, v2=header, crc=crc) +end + +function encodedpage(payload::Vector{UInt8}, valuecount::Integer, encoding; v2::Bool=false, + levels=nothing, maxlevel::Int=0, repetitions=nothing, maxrepetition::Int=0, + rows=nothing) + if !v2 + repetition = repetitions === nothing ? UInt8[] : + columnlevels(repetitions, maxrepetition) + definitions = levels === nothing ? UInt8[] : columnlevels(levels, maxlevel) + return columnpage(vcat(repetition, definitions, payload); + v1=columnv1(valuecount; encoding=encoding)) + end + repetition = repetitions === nothing ? UInt8[] : + Parquet.encode_hybrid(UInt64.(repetitions), Parquet._levelbitwidth(maxrepetition)) + definitions = levels === nothing ? UInt8[] : + Parquet.encode_hybrid(UInt64.(levels), Parquet._levelbitwidth(maxlevel)) + nulls = levels === nothing ? 0 : count(level -> level != maxlevel, levels) + rowcount = something(rows, repetitions === nothing ? valuecount : count(iszero, repetitions)) + header = MD.DataPageHeaderV2(num_values=Int32(valuecount), num_nulls=Int32(nulls), + num_rows=Int32(rowcount), encoding=encoding, + definition_levels_byte_length=Int32(length(definitions)), + repetition_levels_byte_length=Int32(length(repetition)), is_compressed=false) + return columnpage(vcat(repetition, definitions, payload); + type=MD.PageType.DATA_PAGE_V2, v2=header) +end + +function rawv2page(payload::Vector{UInt8}; values=1, nulls=0, rows=values, + definition=0, repetition=0, encoding=MD.Encoding.PLAIN, is_compressed=false, + compressed=length(payload), uncompressed=compressed, crc=:valid) + header = MD.DataPageHeaderV2(num_values=Int32(values), num_nulls=Int32(nulls), + num_rows=Int32(rows), encoding=encoding, + definition_levels_byte_length=Int32(definition), + repetition_levels_byte_length=Int32(repetition), is_compressed=is_compressed) + return columnpage(payload; type=MD.PageType.DATA_PAGE_V2, v2=header, + compressed=compressed, uncompressed=uncompressed, crc=crc) +end + +function columnroot(children::Int) + return MD.SchemaElement(name="root", num_children=Int32(children)) +end + +# Build a complete single-column file: magic, pages, footer, footer length, magic. +function syntheticfile(pages::Vector{Vector{UInt8}}, leaf::MD.SchemaElement; num_values::Integer, + codec=MD.CompressionCodec.UNCOMPRESSED, group=nothing, data_page_offset=4, + index_page_offset=nothing, dictionary_page_offset=nothing, + total=nothing, file_path=nothing, crypto=nothing, encryptedmeta=nothing, type=leaf.type_, path=nothing, + extrachunks=0, rows=num_values, schemaelements=nothing) + body = isempty(pages) ? UInt8[] : vcat(pages...) + size = something(total, length(body)) + columnpath = something(path, group === nothing ? ["value"] : [group.name, "value"]) + md = MD.ColumnMetaData(type_=type, encodings=[MD.Encoding.PLAIN, MD.Encoding.RLE], path_in_schema=columnpath, + codec=codec, num_values=Int64(num_values), total_uncompressed_size=Int64(size), total_compressed_size=Int64(size), + data_page_offset=Int64(data_page_offset), + index_page_offset=index_page_offset, + dictionary_page_offset=dictionary_page_offset) + chunk = MD.ColumnChunk(meta_data=md, file_path=file_path, crypto_metadata=crypto, encrypted_column_metadata=encryptedmeta) + elements = something(schemaelements, + group === nothing ? [columnroot(1), leaf] : [columnroot(1), group, leaf]) + rowgroup = MD.RowGroup(columns=fill(chunk, 1 + extrachunks), + total_byte_size=Int64(size), num_rows=Int64(rows)) + meta = MD.FileMetaData(version=Int32(1), schema=elements, num_rows=Int64(rows), + row_groups=[rowgroup]) + footer = TH.encode(meta) + magic = UInt8[0x50, 0x41, 0x52, 0x31] + bytes = vcat(magic, body, footer, reinterpret(UInt8, [htol(UInt32(length(footer)))]), magic) + return bytes, meta, Parquet.Schema(meta) +end + +function readsynthetic(pages, leaf; limits=Parquet.Limits(), kwargs...) + bytes, meta, schema = syntheticfile(pages, leaf; kwargs...) + file = Parquet.File(bytes) + values = Parquet.readcolumn(file, meta, schema, 1, 1; limits=limits) + close(file) + return values +end + +function readsyntheticstream(pages, leaf; expected_rows=nothing, + limits=Parquet.Limits(), kwargs...) + bytes, meta, schema = syntheticfile(pages, leaf; kwargs...) + file = Parquet.File(bytes) + stream = Parquet.readleafstream(file, meta, schema, 1, 1; + expected_rows=expected_rows, limits=limits) + close(file) + return stream +end + +function listleafschema(type; width=nothing) + leaf = MD.SchemaElement(name="element", type_=type, + repetition_type=MD.FieldRepetitionType.OPTIONAL, + type_length=width === nothing ? nothing : Int32(width)) + root = columnroot(1) + outer = MD.SchemaElement(name="items", + repetition_type=MD.FieldRepetitionType.OPTIONAL, num_children=Int32(1)) + repeated = MD.SchemaElement(name="list", + repetition_type=MD.FieldRepetitionType.REPEATED, num_children=Int32(1)) + return leaf, [root, outer, repeated, leaf] +end + +function columnbitpacked(values, maxlevel::Int) + width = Parquet._levelbitwidth(maxlevel) + output = zeros(UInt8, cld(length(values) * width, 8)) + for (index, rawvalue) in enumerate(values) + value = UInt64(rawvalue) + for bit in 0:(width - 1) + source = width - bit - 1 + iszero(value & (UInt64(1) << source)) && continue + absolute = (index - 1) * width + bit + output[(absolute >> 3) + 1] |= UInt8(1) << (7 - (absolute & 7)) + end + end + return output +end + +@testset "leaf stream invariants" begin + repetition = UInt64[0, 0, 0, 0, 1, 1, 0] + definition = UInt64[0, 1, 2, 3, 2, 3, 3] + values = Int32[0, -1, 11016] + stream = Parquet.LeafStream(repetition, definition, values, 1, 3; + expected_rows=5) + @test stream.repetition === repetition + @test stream.definition === definition + @test stream.values === values + @test length(stream) == 7 && !isempty(stream) + @test isempty(Parquet.LeafStream(UInt64[], UInt64[], Int32[], 1, 3; + expected_rows=0)) + @test_throws Parquet.FormatError Parquet.LeafStream(UInt64[0], UInt64[], + Int32[], 1, 3) + @test_throws Parquet.FormatError Parquet.LeafStream(UInt64[0], UInt64[3], + Int32[], 1, 3) + @test_throws Parquet.FormatError Parquet.LeafStream(UInt64[0], UInt64[4], + Int32[1], 1, 3) + @test_throws Parquet.FormatError Parquet.LeafStream(UInt64[2], UInt64[3], + Int32[1], 1, 3) + @test_throws Parquet.FormatError Parquet.LeafStream(UInt64[1], UInt64[3], + Int32[1], 1, 3) + @test_throws Parquet.FormatError Parquet.LeafStream(UInt64[0, 1], UInt64[0, 0], + Int32[], 1, 3) + @test_throws Parquet.FormatError Parquet.LeafStream(repetition, definition, + values, 1, 3; expected_rows=4) + @test_throws Parquet.FormatError Parquet.LeafStream(UInt64[], UInt64[], + Int32[], -1, 3) +end + +@testset "nested V1 leaf streams" begin + leaf, elements = listleafschema(MD.Type.INT32) + path = ["items", "list", "element"] + repetition = [0, 0, 0, 0, 1, 1, 0] + definition = [0, 1, 2, 3, 2, 3, 3] + values = Int32[0, -1, 11016] + page = datapage(values; levels=definition, maxlevel=3, + repetitions=repetition, maxrepetition=1) + stream = readsyntheticstream([page], leaf; num_values=7, rows=5, + path=path, schemaelements=elements, expected_rows=5) + @test stream.repetition == repetition + @test stream.definition == definition + @test stream.values == values + @test_throws Parquet.FormatError readsynthetic([page], leaf; num_values=7, + rows=5, path=path, schemaelements=elements) + + packedpayload = vcat(columnbitpacked(repetition, 1), + columnbitpacked(definition, 3), columnplain(values)) + packedheader = columnv1(7; levelencoding=MD.Encoding.BIT_PACKED, + repetitionencoding=MD.Encoding.BIT_PACKED) + packedpage = columnpage(packedpayload; v1=packedheader) + packed = readsyntheticstream([packedpage], leaf; num_values=7, rows=5, + path=path, schemaelements=elements, expected_rows=5) + @test packed.repetition == repetition + @test packed.definition == definition + @test packed.values == values + + pages = [ + datapage(Int32[1, 2]; levels=[3, 3], maxlevel=3, + repetitions=[0, 1], maxrepetition=1), + datapage(Int32[3, 4]; levels=[3, 3], maxlevel=3, + repetitions=[1, 0], maxrepetition=1), + ] + split = readsyntheticstream(pages, leaf; num_values=4, rows=2, + path=path, schemaelements=elements, expected_rows=2) + @test split.repetition == [0, 1, 1, 0] + @test split.definition == fill(UInt64(3), 4) + @test split.values == Int32[1, 2, 3, 4] + + badstart = datapage(Int32[1]; levels=[3], maxlevel=3, + repetitions=[1], maxrepetition=1) + @test_throws Parquet.FormatError readsyntheticstream([badstart], leaf; + num_values=1, rows=0, path=path, schemaelements=elements) + @test_throws Parquet.FormatError readsyntheticstream([page], leaf; + num_values=7, rows=5, path=path, schemaelements=elements, expected_rows=4) +end + +@testset "nested V2 leaf streams" begin + leaf, elements = listleafschema(MD.Type.INT32) + path = ["items", "list", "element"] + repetition = [0, 0, 0, 0, 1, 1, 0] + definition = [0, 1, 2, 3, 2, 3, 3] + values = Int32[0, -1, 11016] + repetitionbytes = Parquet.encode_hybrid(UInt64.(repetition), 1) + page = datapagev2(values; levels=definition, maxlevel=3, + repetition=repetitionbytes, rows=5) + stream = readsyntheticstream([page], leaf; num_values=7, rows=5, + path=path, schemaelements=elements, expected_rows=5) + @test stream.repetition == repetition + @test stream.definition == definition + @test stream.values == values + + firstrepetition = Parquet.encode_hybrid(UInt64[0, 1], 1) + secondrepetition = Parquet.encode_hybrid(UInt64[0, 1, 1], 1) + pages = [ + datapagev2(Int32[1, 2]; levels=[3, 3], maxlevel=3, + repetition=firstrepetition, rows=1), + datapagev2(Int32[3, 4, 5]; levels=[3, 3, 3], maxlevel=3, + repetition=secondrepetition, rows=1), + ] + split = readsyntheticstream(pages, leaf; num_values=5, rows=2, + path=path, schemaelements=elements, expected_rows=2) + @test split.repetition == [0, 1, 0, 1, 1] + @test split.values == Int32[1, 2, 3, 4, 5] + + nonzero = Parquet.encode_hybrid(UInt64[1], 1) + badstart = datapagev2(Int32[1]; levels=[3], maxlevel=3, + repetition=nonzero, rows=0) + @test_throws Parquet.FormatError readsyntheticstream([badstart], leaf; + num_values=1, rows=0, path=path, schemaelements=elements) + wrongrows = datapagev2(Int32[1, 2]; levels=[3, 3], maxlevel=3, + repetition=firstrepetition, rows=2) + @test_throws Parquet.FormatError readsyntheticstream([wrongrows], leaf; + num_values=2, rows=2, path=path, schemaelements=elements) + wrongnulls = datapagev2(Int32[1]; levels=[3, 2], maxlevel=3, + repetition=firstrepetition, rows=1, nulls=0) + @test_throws Parquet.FormatError readsyntheticstream([wrongnulls], leaf; + num_values=2, rows=1, path=path, schemaelements=elements) + trailingrepetition = datapagev2(Int32[1, 2]; levels=[3, 3], maxlevel=3, + repetition=vcat(firstrepetition, UInt8[0x00]), rows=1) + @test_throws Parquet.FormatError readsyntheticstream([trailingrepetition], leaf; + num_values=2, rows=1, path=path, schemaelements=elements) + @test_throws Parquet.FormatError readsyntheticstream([page], leaf; + num_values=7, rows=5, path=path, schemaelements=elements, expected_rows=4) +end + +@testset "nested leaf value encodings" begin + repetition = [0, 0, 0, 1] + definition = [3, 0, 3, 3] + cases = ( + (Int32[1, -2, 3], MD.Encoding.DELTA_BINARY_PACKED, + Parquet.encode_delta_binary_packed(Int32[1, -2, 3]), MD.Type.INT32, nothing), + (Int64[1, -2, 3], MD.Encoding.DELTA_BINARY_PACKED, + Parquet.encode_delta_binary_packed(Int64[1, -2, 3]), MD.Type.INT64, nothing), + ([UInt8[0x61], UInt8[], UInt8[0x62, 0x63]], + MD.Encoding.DELTA_LENGTH_BYTE_ARRAY, + Parquet.encode_delta_length_byte_array( + [UInt8[0x61], UInt8[], UInt8[0x62, 0x63]]), + MD.Type.BYTE_ARRAY, nothing), + ([collect(codeunits(value)) for value in ("prefix-a", "prefix-b", "prefix-bb")], + MD.Encoding.DELTA_BYTE_ARRAY, + Parquet.encode_delta_byte_array(["prefix-a", "prefix-b", "prefix-bb"]), + MD.Type.BYTE_ARRAY, nothing), + (Float32[-0.0, 1.5, Inf], MD.Encoding.BYTE_STREAM_SPLIT, + Parquet.encode_byte_stream_split(Float32[-0.0, 1.5, Inf]), + MD.Type.FLOAT, nothing), + (Bool[true, false, true], MD.Encoding.RLE, + Parquet.encode_hybrid(UInt64[1, 0, 1], 1; length_prefix=true), + MD.Type.BOOLEAN, nothing), + ) + for v2 in (false, true), (expected, encoding, payload, type, width) in cases + leaf, elements = listleafschema(type; width=width) + page = encodedpage(payload, 4, encoding; v2=v2, levels=definition, + maxlevel=3, repetitions=repetition, maxrepetition=1) + stream = readsyntheticstream([page], leaf; num_values=4, rows=3, + path=["items", "list", "element"], schemaelements=elements, + expected_rows=3) + @test stream.repetition == repetition + @test stream.definition == definition + if expected isa Vector{Float32} + @test reinterpret(UInt32, stream.values) == reinterpret(UInt32, expected) + else + @test stream.values == expected + end + end + + fixedvalues = UInt8[1 4 7; 2 5 8; 3 6 9] + expectedfixed = [fixedvalues[:, index] for index in axes(fixedvalues, 2)] + for encoding in (MD.Encoding.DELTA_BYTE_ARRAY, MD.Encoding.BYTE_STREAM_SPLIT), + v2 in (false, true) + payload = encoding == MD.Encoding.DELTA_BYTE_ARRAY ? + Parquet.encode_delta_byte_array_fixed(fixedvalues) : + Parquet.encode_byte_stream_split_fixed(fixedvalues) + leaf, elements = listleafschema(MD.Type.FIXED_LEN_BYTE_ARRAY; width=3) + page = encodedpage(payload, 4, encoding; v2=v2, levels=definition, + maxlevel=3, repetitions=repetition, maxrepetition=1) + stream = readsyntheticstream([page], leaf; num_values=4, rows=3, + path=["items", "list", "element"], schemaelements=elements, + expected_rows=3) + @test stream.values == expectedfixed + end + + dictionaryvalues = Int32[10, 20] + dictionarypayload = columnplain(dictionaryvalues) + dictionaryheader = MD.DictionaryPageHeader(num_values=Int32(2), + encoding=MD.Encoding.PLAIN) + dictionary = columnpage(dictionarypayload; type=MD.PageType.DICTIONARY_PAGE, + dict=dictionaryheader) + indices = vcat(UInt8[0x01], Parquet.encode_hybrid(UInt64[1, 0, 1], 1)) + leaf, elements = listleafschema(MD.Type.INT32) + for v2 in (false, true) + data = encodedpage(indices, 4, MD.Encoding.RLE_DICTIONARY; v2=v2, + levels=definition, maxlevel=3, repetitions=repetition, maxrepetition=1) + stream = readsyntheticstream([dictionary, data], leaf; num_values=4, rows=3, + path=["items", "list", "element"], schemaelements=elements, + expected_rows=3, dictionary_page_offset=Int64(4), + data_page_offset=4 + length(dictionary)) + @test stream.values == Int32[20, 10, 20] + end +end + +@testset "leaf stream materialized ownership" begin + nvalues = 100 + wide = fill(UInt8(0x78), 4000) + tiny = UInt8[0x61] + dictionarypayload = columnplain([wide, tiny]) + dictionaryheader = MD.DictionaryPageHeader(num_values=Int32(2), + encoding=MD.Encoding.PLAIN) + dictionary = columnpage(dictionarypayload; + type=MD.PageType.DICTIONARY_PAGE, dict=dictionaryheader) + indices = vcat(UInt8[0x01], + Parquet.encode_hybrid(fill(UInt64(1), nvalues), 1)) + data = encodedpage(indices, nvalues, MD.Encoding.RLE_DICTIONARY) + leaf = columnleaf(MD.Type.BYTE_ARRAY) + offset = 4 + length(dictionary) + limits = Parquet.Limits(max_materialized_bytes=100_000) + values = readsynthetic([dictionary, data], leaf; num_values=nvalues, + dictionary_page_offset=Int64(4), data_page_offset=offset, + limits=limits) + @test values == fill(tiny, nvalues) + + bytes, metadata, schema = syntheticfile([dictionary, data], leaf; + num_values=nvalues, dictionary_page_offset=Int64(4), + data_page_offset=offset) + file = Parquet.File(bytes) + budget = Parquet._LiveByteBudget(Parquet.Limits()) + stream = Parquet.readleafstream(file, metadata, schema, 1, 1; + budget=budget) + retained = Parquet._leafretainedbytes(Vector{UInt8}, nvalues, + stream.values) + @test Parquet._budgetused(budget) == retained + close(file) + + optional = columnleaf(MD.Type.BYTE_ARRAY; + repetition=MD.FieldRepetitionType.OPTIONAL) + optionalpage = datapage([UInt8[0x61], UInt8[0x62, 0x63]]; + levels=[1, 0, 1], maxlevel=1) + bytes, metadata, schema = syntheticfile([optionalpage], optional; + num_values=3) + file = Parquet.File(bytes) + flatbudget = Parquet._LiveByteBudget(Parquet.Limits()) + output = Parquet.readcolumn(file, metadata, schema, 1, 1; + budget=flatbudget) + @test isequal(output, + Union{Missing,Vector{UInt8}}[UInt8[0x61], missing, + UInt8[0x62, 0x63]]) + expected = Parquet._materializedarraybytes(eltype(output), length(output)) + expected += Parquet._leafchildbytes(Vector{UInt8}, + collect(skipmissing(output))) + @test Parquet._budgetused(flatbudget) == expected + close(file) + + delta = Parquet.encode_delta_binary_packed(Int32[1]) + malformed = encodedpage(vcat(delta, UInt8[0x00]), 1, + MD.Encoding.DELTA_BINARY_PACKED) + bytes, metadata, schema = syntheticfile([malformed], + columnleaf(MD.Type.INT32); num_values=1) + file = Parquet.File(bytes) + failurebudget = Parquet._LiveByteBudget(Parquet.Limits()) + @test_throws Parquet.FormatError Parquet.readleafstream(file, metadata, + schema, 1, 1; budget=failurebudget) + @test Parquet._budgetused(failurebudget) == 0 + close(file) + + deltalimits = Parquet.Limits(max_materialized_bytes= + Parquet._materializedarraybytes(Int32, 1) - 1) + function rejectintdelta() + deltabudget = Parquet._LiveByteBudget(deltalimits) + @test_throws Parquet.LimitError Parquet._decodeencodedvalues(Int32, + MD.Encoding.DELTA_BINARY_PACKED, delta, 1, nothing, 1, + deltalimits, deltabudget) + @test Parquet._budgetused(deltabudget) == 0 + return + end + rejectintdelta() + GC.gc() + @test @allocated(rejectintdelta()) < 10_000 +end + +@testset "required flat PLAIN columns" begin + int32 = columnleaf(MD.Type.INT32) + values = readsynthetic([datapage(Int32[1, -2, 3])], int32; num_values=3) + @test values == Int32[1, -2, 3] && values isa Vector{Int32} + two = readsynthetic([datapage(Int32[1, 2]), datapage(Int32[3]), datapage(Int32[4, 5, 6])], int32; num_values=6) + @test two == Int32[1, 2, 3, 4, 5, 6] + @test readsynthetic(Vector{UInt8}[], int32; num_values=0, + data_page_offset=0) == Int32[] + @test readsynthetic([datapage(Int32[])], int32; num_values=0) == Int32[] + @test readsynthetic([datapage(Int32[]), datapage(Int32[9])], int32; num_values=1) == Int32[9] + @test readsynthetic([datapage(Int32[5]; crc=:none)], int32; num_values=1) == Int32[5] + # parquet-mr marks the absent level stream of required columns as BIT_PACKED + @test readsynthetic([datapage(Int32[5]; levelencoding=MD.Encoding.BIT_PACKED)], int32; num_values=1) == Int32[5] + bools = Bool[true, false, true, true, false, false, false, true, true] + @test readsynthetic([datapage(bools)], columnleaf(MD.Type.BOOLEAN); num_values=9) == bools + @test readsynthetic([datapage(Int64[typemin(Int64), 0, typemax(Int64)])], columnleaf(MD.Type.INT64); num_values=3) == Int64[typemin(Int64), 0, typemax(Int64)] + floats = Float32[-0.0, NaN, Inf, 1.5] + decoded = readsynthetic([datapage(floats)], columnleaf(MD.Type.FLOAT); num_values=4) + @test reinterpret(UInt32, decoded) == reinterpret(UInt32, floats) + doubles = [1.0, -2.5, NaN] + @test isequal(readsynthetic([datapage(doubles)], columnleaf(MD.Type.DOUBLE); num_values=3), doubles) + strings = [UInt8[], collect(codeunits("parquet")), UInt8[0x00, 0xff]] + @test readsynthetic([datapage(strings)], columnleaf(MD.Type.BYTE_ARRAY); num_values=3) == strings + fixed = [UInt8[1, 2, 3], UInt8[4, 5, 6]] + decodedfixed = readsynthetic([datapage(fixed; width=3)], columnleaf(MD.Type.FIXED_LEN_BYTE_ARRAY; width=3); num_values=2) + @test decodedfixed == fixed && decodedfixed isa Vector{Vector{UInt8}} +end + +@testset "optional flat PLAIN columns" begin + optional = MD.FieldRepetitionType.OPTIONAL + int32 = columnleaf(MD.Type.INT32; repetition=optional) + values = readsynthetic([datapage(Int32[10, 20, 30]; levels=[1, 0, 1, 1, 0], maxlevel=1)], int32; num_values=5) + @test isequal(values, Union{Missing,Int32}[10, missing, 20, 30, missing]) && values isa Vector{Union{Missing,Int32}} + allnull = readsynthetic([datapage(Int32[]; levels=[0, 0, 0], maxlevel=1)], int32; num_values=3) + @test all(ismissing, allnull) && length(allnull) == 3 + allpresent = readsynthetic([datapage(Int32[1, 2]; levels=[1, 1], maxlevel=1)], int32; num_values=2) + @test isequal(allpresent, Union{Missing,Int32}[1, 2]) + pages = [datapage(Int32[1]; levels=[0, 1], maxlevel=1), datapage(Int32[]; levels=[0], maxlevel=1), datapage(Int32[2, 3]; levels=[1, 1], maxlevel=1)] + @test isequal(readsynthetic(pages, int32; num_values=5), Union{Missing,Int32}[missing, 1, missing, 2, 3]) + @test isequal(readsynthetic([datapage(Int32[]; levels=Int[], maxlevel=1)], int32; num_values=0), Union{Missing,Int32}[]) + bools = readsynthetic([datapage(Bool[true, false, true]; levels=[1, 0, 1, 1], maxlevel=1)], columnleaf(MD.Type.BOOLEAN; repetition=optional); num_values=4) + @test isequal(bools, Union{Missing,Bool}[true, missing, false, true]) + strings = readsynthetic([datapage([UInt8[0x61], UInt8[]]; levels=[0, 1, 1], maxlevel=1)], columnleaf(MD.Type.BYTE_ARRAY; repetition=optional); num_values=3) + @test isequal(strings, Union{Missing,Vector{UInt8}}[missing, UInt8[0x61], UInt8[]]) + fixed = readsynthetic([datapage([UInt8[9, 9]]; levels=[1, 0], maxlevel=1, width=2)], columnleaf(MD.Type.FIXED_LEN_BYTE_ARRAY; repetition=optional, width=2); num_values=2) + @test isequal(fixed, Union{Missing,Vector{UInt8}}[UInt8[9, 9], missing]) + doubles = readsynthetic([datapage([NaN, 0.5]; levels=[1, 0, 1], maxlevel=1)], columnleaf(MD.Type.DOUBLE; repetition=optional); num_values=3) + @test isequal(doubles, Union{Missing,Float64}[NaN, missing, 0.5]) + # a flat leaf under an optional group has max definition level 2 + group = MD.SchemaElement(name="g", repetition_type=optional, num_children=Int32(1)) + nested = readsynthetic([datapage(Int32[7, 8]; levels=[2, 1, 0, 2], maxlevel=2)], int32; num_values=4, group=group) + @test isequal(nested, Union{Missing,Int32}[7, missing, missing, 8]) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[7]; levels=[2, 3], maxlevel=2)], int32; num_values=2, group=group) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[7]; levels=[3], maxlevel=2)], int32; num_values=1, group=group) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[7]; levels=[1], maxlevel=1, levelencoding=MD.Encoding.BIT_PACKED)], int32; num_values=1) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[7, 8]; levels=[1, 0], maxlevel=1)], int32; num_values=2) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[]; levels=[1], maxlevel=1)], int32; num_values=1) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[1]; levels=[1], maxlevel=1, extra=UInt8[0x00])], int32; num_values=1) + levelsonly = columnpage(columnlevels([1], 1)[1:(end - 1)]; v1=columnv1(1)) + @test_throws Parquet.FormatError readsynthetic([levelsonly], int32; num_values=1) +end + +@testset "flat Data Page V2 columns" begin + required = columnleaf(MD.Type.INT32) + optional = columnleaf(MD.Type.INT32; repetition=MD.FieldRepetitionType.OPTIONAL) + @test readsynthetic([datapagev2(Int32[1, -2, 3])], required; num_values=3) == Int32[1, -2, 3] + values = readsynthetic([datapagev2(Int32[10, 20]; levels=[1, 0, 1], maxlevel=1)], + optional; num_values=3) + @test isequal(values, Union{Missing,Int32}[10, missing, 20]) + allnull = readsynthetic([datapagev2(Int32[]; levels=[0, 0], maxlevel=1)], + optional; num_values=2) + @test isequal(allnull, Union{Missing,Int32}[missing, missing]) + @test readsynthetic([datapagev2(Int32[])], required; num_values=0) == Int32[] + zero_levels = Parquet.encode_hybrid(zeros(UInt64, 3), 0) + redundant = datapagev2(Int32[4, 5, 6]; repetition=zero_levels, definition=zero_levels) + @test readsynthetic([redundant], required; num_values=3) == Int32[4, 5, 6] + absentdefault = datapagev2(Int32[9]; is_compressed=nothing) + @test readsynthetic([absentdefault], required; num_values=1) == Int32[9] + + raw = Parquet.encode_plain(Int32[7, 8]) + compressed = Parquet.compress(MD.CompressionCodec.SNAPPY, raw) + compressedheader = MD.DataPageHeaderV2(num_values=Int32(2), num_nulls=Int32(0), + num_rows=Int32(2), encoding=MD.Encoding.PLAIN, + definition_levels_byte_length=Int32(0), repetition_levels_byte_length=Int32(0)) + compressedpage = columnpage(compressed; type=MD.PageType.DATA_PAGE_V2, + v2=compressedheader, uncompressed=length(raw)) + @test readsynthetic([compressedpage], required; num_values=2, + codec=MD.CompressionCodec.SNAPPY) == Int32[7, 8] + uncompressedheader = MD.DataPageHeaderV2(num_values=Int32(2), num_nulls=Int32(0), + num_rows=Int32(2), encoding=MD.Encoding.PLAIN, + definition_levels_byte_length=Int32(0), repetition_levels_byte_length=Int32(0), + is_compressed=false) + uncompressedpage = columnpage(raw; type=MD.PageType.DATA_PAGE_V2, v2=uncompressedheader) + @test readsynthetic([uncompressedpage], required; num_values=2, + codec=MD.CompressionCodec.SNAPPY) == Int32[7, 8] + emptyheader = MD.DataPageHeaderV2(num_values=Int32(0), num_nulls=Int32(0), + num_rows=Int32(0), encoding=MD.Encoding.PLAIN, + definition_levels_byte_length=Int32(0), repetition_levels_byte_length=Int32(0)) + emptypage = columnpage(UInt8[]; type=MD.PageType.DATA_PAGE_V2, v2=emptyheader) + @test readsynthetic([emptypage], required; num_values=0, + codec=MD.CompressionCodec.SNAPPY) == Int32[] +end + +@testset "page value encoding dispatch" begin + bytearray = columnleaf(MD.Type.BYTE_ARRAY) + fixed = columnleaf(MD.Type.FIXED_LEN_BYTE_ARRAY; width=3) + cases = ( + (Int32[1, 2, -5, 8], MD.Encoding.DELTA_BINARY_PACKED, + Parquet.encode_delta_binary_packed(Int32[1, 2, -5, 8]), columnleaf(MD.Type.INT32)), + (Int64[typemin(Int64), -1, 0, typemax(Int64)], MD.Encoding.DELTA_BINARY_PACKED, + Parquet.encode_delta_binary_packed(Int64[typemin(Int64), -1, 0, typemax(Int64)]), + columnleaf(MD.Type.INT64)), + ([UInt8[0x61], UInt8[0x62, 0x63], UInt8[]], MD.Encoding.DELTA_LENGTH_BYTE_ARRAY, + Parquet.encode_delta_length_byte_array([UInt8[0x61], UInt8[0x62, 0x63], UInt8[]]), bytearray), + ([collect(codeunits(value)) for value in ("prefix-a", "prefix-b", "prefix-bb")], + MD.Encoding.DELTA_BYTE_ARRAY, + Parquet.encode_delta_byte_array(["prefix-a", "prefix-b", "prefix-bb"]), bytearray), + (Float32[-0.0, 1.5, Inf], MD.Encoding.BYTE_STREAM_SPLIT, + Parquet.encode_byte_stream_split(Float32[-0.0, 1.5, Inf]), columnleaf(MD.Type.FLOAT)), + ) + for v2 in (false, true), (expected, encoding, payload, leaf) in cases + page = encodedpage(payload, length(expected), encoding; v2=v2) + actual = readsynthetic([page], leaf; num_values=length(expected)) + if expected isa Vector{Float32} + @test reinterpret(UInt32, actual) == reinterpret(UInt32, expected) + else + @test actual == expected + end + end + fixedvalues = UInt8[1 4; 2 5; 3 6] + for (encoding, payload) in ( + (MD.Encoding.DELTA_BYTE_ARRAY, Parquet.encode_delta_byte_array_fixed(fixedvalues)), + (MD.Encoding.BYTE_STREAM_SPLIT, Parquet.encode_byte_stream_split_fixed(fixedvalues))) + for v2 in (false, true) + page = encodedpage(payload, 2, encoding; v2=v2) + @test readsynthetic([page], fixed; num_values=2) == [UInt8[1, 2, 3], UInt8[4, 5, 6]] + end + end + bools = Bool[true, false, true, true, false] + rle = Parquet.encode_hybrid(UInt64.(bools), 1; length_prefix=true) + for v2 in (false, true) + page = encodedpage(rle, length(bools), MD.Encoding.RLE; v2=v2) + @test readsynthetic([page], columnleaf(MD.Type.BOOLEAN); num_values=length(bools)) == bools + end + optional = columnleaf(MD.Type.INT32; repetition=MD.FieldRepetitionType.OPTIONAL) + values = Int32[10, 20, 30] + levels = [1, 0, 1, 1] + page = encodedpage(Parquet.encode_delta_binary_packed(values), 4, + MD.Encoding.DELTA_BINARY_PACKED; v2=true, levels=levels, maxlevel=1) + @test isequal(readsynthetic([page], optional; num_values=4), + Union{Missing,Int32}[10, missing, 20, 30]) +end + +@testset "Data Page V2 malformed input" begin + required = columnleaf(MD.Type.INT32) + optional = columnleaf(MD.Type.INT32; repetition=MD.FieldRepetitionType.OPTIONAL) + value = Parquet.encode_plain(Int32[1]) + @test_throws Parquet.FormatError readsynthetic([rawv2page(value; values=1, rows=0)], + required; num_values=1) + @test_throws Parquet.FormatError readsynthetic([rawv2page(value; values=1, nulls=1)], + required; num_values=1) + definitions = Parquet.encode_hybrid(UInt64[1, 0], 1) + mismatch = rawv2page(vcat(definitions, value); values=2, nulls=0, rows=2, + definition=length(definitions)) + @test_throws Parquet.FormatError readsynthetic([mismatch], optional; num_values=2) + trailinglevels = rawv2page(vcat(definitions, UInt8[0x00], value); values=2, + nulls=1, rows=2, definition=length(definitions) + 1) + @test_throws Parquet.FormatError readsynthetic([trailinglevels], optional; num_values=2) + badrepetition = rawv2page(vcat(UInt8[0x00], value); repetition=1) + @test_throws Parquet.FormatError readsynthetic([badrepetition], required; num_values=1) + trailingvalue = rawv2page(vcat(value, UInt8[0x00])) + @test_throws Parquet.FormatError readsynthetic([trailingvalue], required; num_values=1) + shortvalue = rawv2page(value[1:(end - 1)]; uncompressed=length(value) - 1) + @test_throws Parquet.FormatError readsynthetic([shortvalue], required; num_values=1) + corruptcompressed = rawv2page(UInt8[0xff]; is_compressed=true, uncompressed=4) + @test_throws Parquet.FormatError readsynthetic([corruptcompressed], required; + num_values=1, codec=MD.CompressionCodec.SNAPPY) + wrongtype = rawv2page(Parquet.encode_delta_binary_packed(Int32[1]); + encoding=MD.Encoding.DELTA_BINARY_PACKED) + @test_throws Parquet.FormatError readsynthetic([wrongtype], + columnleaf(MD.Type.FLOAT); num_values=1) + unknown = rawv2page(value; encoding=MD.Encoding.T(42)) + @test_throws Parquet.FormatError readsynthetic([unknown], required; num_values=1) + corruptcrc = datapagev2(Int32[1]) + corruptcrc[end] ⊻= 0x01 + @test_throws Parquet.FormatError readsynthetic([corruptcrc], required; num_values=1) +end + +@testset "exact consumption and unsupported pages" begin + int32 = columnleaf(MD.Type.INT32) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[1]; extra=UInt8[0x00])], int32; num_values=1) + short = columnpage(Parquet.encode_plain(Int32[1, 2])[1:(end - 1)]; v1=columnv1(2)) + @test_throws Parquet.FormatError readsynthetic([short], int32; num_values=2) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[1])], int32; num_values=1, codec=MD.CompressionCodec.SNAPPY) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[1])], int32; num_values=1, codec=MD.CompressionCodec.T(42)) + for encoding in (MD.Encoding.PLAIN_DICTIONARY, MD.Encoding.RLE_DICTIONARY, MD.Encoding.DELTA_BINARY_PACKED, MD.Encoding.T(10)) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[1]; encoding=encoding)], int32; num_values=1) + end + dictionary = columnpage(Parquet.encode_plain(Int32[1]); type=MD.PageType.DICTIONARY_PAGE, + dict=MD.DictionaryPageHeader(num_values=Int32(1), encoding=MD.Encoding.PLAIN)) + data = datapage(Int32[1]) + @test readsynthetic([dictionary, data], int32; num_values=1, + data_page_offset=4 + length(dictionary), dictionary_page_offset=4) == + Int32[1] + @test_throws Parquet.FormatError readsynthetic([data, dictionary], int32; + num_values=1, data_page_offset=4, + dictionary_page_offset=4 + length(data)) + v2 = columnpage(Parquet.encode_plain(Int32[1]); type=MD.PageType.DATA_PAGE_V2, + v2=MD.DataPageHeaderV2(num_values=Int32(1), num_nulls=Int32(0), num_rows=Int32(1), encoding=MD.Encoding.PLAIN, + definition_levels_byte_length=Int32(0), repetition_levels_byte_length=Int32(0))) + @test readsynthetic([v2], int32; num_values=1) == Int32[1] + # index pages and unknown page types are framed, CRC-checked, and skipped + index = columnpage(UInt8[0x00]; type=MD.PageType.INDEX_PAGE, + index=MD.IndexPageHeader()) + @test readsynthetic([index, data, index], int32; num_values=1, + data_page_offset=4 + length(index), index_page_offset=4) == Int32[1] + unknown = columnpage(UInt8[0x01, 0x02]; type=MD.PageType.T(9)) + @test readsynthetic([datapage(Int32[4]), unknown], int32; + num_values=1) == Int32[4] + badindex = copy(index) + badindex[end] ⊻= 0x01 + @test_throws Parquet.FormatError readsynthetic([badindex, data], int32; + num_values=1, data_page_offset=4 + length(index), + index_page_offset=4) + truncatedindex = columnpage(UInt8[0x00]; + type=MD.PageType.INDEX_PAGE, index=MD.IndexPageHeader(), + compressed=2) + @test_throws Parquet.FormatError readsynthetic([truncatedindex, data], + int32; num_values=1, data_page_offset=4 + length(truncatedindex), + index_page_offset=4) + mismatch = columnpage(Parquet.encode_plain(Int32[1]); v1=columnv1(1), uncompressed=5) + @test_throws Parquet.FormatError readsynthetic([mismatch], int32; num_values=1) + corrupt = datapage(Int32[1, 2]) + corrupt[end] ⊻= 0x01 + @test_throws Parquet.FormatError readsynthetic([corrupt], int32; num_values=2) +end + +@testset "physical page frame limits and cleanup" begin + int32 = columnleaf(MD.Type.INT32) + index = columnpage(UInt8[0x10]; type=MD.PageType.INDEX_PAGE, + index=MD.IndexPageHeader()) + unknown = columnpage(UInt8[0x20]; type=MD.PageType.T(9)) + dictionary = columnpage(Parquet.encode_plain(Int32[9]); + type=MD.PageType.DICTIONARY_PAGE, + dict=MD.DictionaryPageHeader(num_values=Int32(1), + encoding=MD.Encoding.PLAIN)) + for data in (datapage(Int32[1]), datapagev2(Int32[1])) + plainframes = [index, unknown, data, unknown] + dataoffset = 4 + length(index) + length(unknown) + exact = Parquet.Limits(max_container_elements=4) + @test readsynthetic(plainframes, int32; num_values=1, + data_page_offset=dataoffset, index_page_offset=4, + limits=exact) == Int32[1] + failure = try + readsynthetic(plainframes, int32; num_values=1, + data_page_offset=dataoffset, index_page_offset=4, + limits=Parquet.Limits(max_container_elements=3)) + nothing + catch err + err + end + @test failure isa Parquet.LimitError + @test failure.resource == :container_elements + @test failure.requested == 4 + @test failure.maximum == 3 + + dictionaryframes = [dictionary, index, unknown, data, unknown] + indexoffset = 4 + length(dictionary) + dataoffset = indexoffset + length(index) + length(unknown) + @test readsynthetic(dictionaryframes, int32; num_values=1, + dictionary_page_offset=4, index_page_offset=indexoffset, + data_page_offset=dataoffset, + limits=Parquet.Limits(max_container_elements=5)) == Int32[1] + failure = try + readsynthetic(dictionaryframes, int32; num_values=1, + dictionary_page_offset=4, index_page_offset=indexoffset, + data_page_offset=dataoffset, + limits=Parquet.Limits(max_container_elements=4)) + nothing + catch err + err + end + @test failure isa Parquet.LimitError + @test failure.requested == 5 + @test failure.maximum == 4 + end + + frames = [dictionary, index, unknown, datapage(Int32[1]), unknown] + bytes, metadata, schema = syntheticfile(frames, int32; num_values=1, + dictionary_page_offset=4, + index_page_offset=4 + length(dictionary), + data_page_offset=4 + length(dictionary) + length(index) + + length(unknown)) + limits = Parquet.Limits(max_container_elements=4) + budget = Parquet._LiveByteBudget(limits) + Parquet._reservearray!(budget, UInt8, 0) + entry = Parquet._budgetused(budget) + file = Parquet.File(bytes) + try + failure = try + Parquet.readcolumn(file, metadata, schema, 1, 1; + limits=limits, budget=budget) + nothing + catch err + err + end + @test failure isa Parquet.LimitError + @test failure.requested == 5 + @test Parquet._budgetused(budget) == entry + finally + close(file) + end + + emptybytes, emptymetadata, emptyschema = syntheticfile(Vector{UInt8}[], + int32; + num_values=0, data_page_offset=0, total=0, rows=0) + emptyfile = Parquet.File(emptybytes) + try + stream = Parquet.readleafstream(emptyfile, emptymetadata, + emptyschema, 1, 1; limits=Parquet.Limits( + max_container_elements=0)) + @test isempty(stream) + finally + close(emptyfile) + end + + emptydictionary = columnpage(UInt8[]; + type=MD.PageType.DICTIONARY_PAGE, + dict=MD.DictionaryPageHeader(num_values=Int32(0), + encoding=MD.Encoding.PLAIN)) + @test isempty(readsynthetic([emptydictionary], int32; num_values=0, + rows=0, data_page_offset=0, dictionary_page_offset=4, + limits=Parquet.Limits(max_container_elements=1))) + dictionarylimit = try + readsynthetic([emptydictionary], int32; num_values=0, rows=0, + data_page_offset=0, dictionary_page_offset=4, + limits=Parquet.Limits(max_container_elements=0)) + nothing + catch err + err + end + @test dictionarylimit isa Parquet.LimitError + @test dictionarylimit.requested == 1 + @test dictionarylimit.maximum == 0 + + negativedictionary = columnpage(UInt8[]; + type=MD.PageType.DICTIONARY_PAGE, + dict=MD.DictionaryPageHeader(num_values=Int32(-1), + encoding=MD.Encoding.PLAIN)) + negativeerror = try + readsynthetic([negativedictionary], int32; num_values=0, rows=0, + data_page_offset=0, dictionary_page_offset=4, + limits=Parquet.Limits(max_container_elements=0)) + nothing + catch err + err + end + @test negativeerror isa Parquet.FormatError + @test negativeerror.message == "negative page value count" + + invaliddictionary = columnpage(Parquet.encode_plain(Int32[9, 10]); + type=MD.PageType.DICTIONARY_PAGE, + dict=MD.DictionaryPageHeader(num_values=Int32(2), + encoding=MD.Encoding.RLE)) + invaliderror = try + readsynthetic([invaliddictionary], int32; num_values=0, rows=0, + data_page_offset=0, dictionary_page_offset=4, + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test invaliderror isa Parquet.FormatError + @test occursin("is not PLAIN", invaliderror.message) + + oversizeddictionary = columnpage(Parquet.encode_plain(Int32[9, 10]); + type=MD.PageType.DICTIONARY_PAGE, + dict=MD.DictionaryPageHeader(num_values=Int32(2), + encoding=MD.Encoding.PLAIN)) + entryerror = try + readsynthetic([oversizeddictionary], int32; num_values=0, rows=0, + data_page_offset=0, dictionary_page_offset=4, + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test entryerror isa Parquet.LimitError + @test entryerror.resource == :container_elements + @test entryerror.requested == 2 + @test entryerror.maximum == 1 + + corrupt = copy(unknown) + corrupt[end] ⊻= 0x01 + checksumframes = [datapage(Int32[1]), corrupt] + checksumerror = try + readsynthetic(checksumframes, int32; num_values=1, + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test checksumerror isa Parquet.FormatError + + mismatchedheader = columnpage(UInt8[]; + type=MD.PageType.INDEX_PAGE, index=MD.IndexPageHeader(), + v1=columnv1(0)) + mismatcherror = try + readsynthetic([datapage(Int32[1]), mismatchedheader], int32; + num_values=1, + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test mismatcherror isa Parquet.FormatError + + truncated = columnpage(UInt8[0x01]; type=MD.PageType.T(9), + compressed=2) + truncationerror = try + readsynthetic([datapage(Int32[1]), truncated], int32; + num_values=1, + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test truncationerror isa Parquet.FormatError + + duplicateerror = try + readsynthetic([dictionary, dictionary, datapage(Int32[1])], int32; + num_values=1, dictionary_page_offset=4, + data_page_offset=4 + 2 * length(dictionary), + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test duplicateerror isa Parquet.FormatError + + falseoffsetframes = [index, unknown, datapage(Int32[1])] + falseoffset = try + readsynthetic(falseoffsetframes, int32; num_values=1, + index_page_offset=4, data_page_offset=4 + length(index), + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test falseoffset isa Parquet.FormatError + + malformed = columnpage(UInt8[]; v1=columnv1(-1)) + malformederror = try + readsynthetic([index, malformed], int32; num_values=0, rows=0, + index_page_offset=4, data_page_offset=4 + length(index), + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test malformederror isa Parquet.FormatError + + invalidcompressed = columnpage(UInt8[0xff]; v1=columnv1(1)) + compressedframes = [index, invalidcompressed] + limitfirst = try + readsynthetic(compressedframes, int32; num_values=1, + index_page_offset=4, data_page_offset=4 + length(index), + codec=MD.CompressionCodec.SNAPPY, + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test limitfirst isa Parquet.LimitError + @test limitfirst.requested == 2 + payloaderror = try + readsynthetic(compressedframes, int32; num_values=1, + index_page_offset=4, data_page_offset=4 + length(index), + codec=MD.CompressionCodec.SNAPPY, + limits=Parquet.Limits(max_container_elements=2)) + nothing + catch err + err + end + @test payloaderror isa Parquet.FormatError + + pagebyteserror = try + readsynthetic([index, datapage(Int32[1])], int32; num_values=1, + index_page_offset=4, data_page_offset=4 + length(index), + limits=Parquet.Limits(max_container_elements=1, + max_page_bytes=3)) + nothing + catch err + err + end + @test pagebyteserror isa Parquet.LimitError + @test pagebyteserror.resource == :page_bytes + @test pagebyteserror.requested == 4 + + bytes, metadata, schema = syntheticfile([datapage(Int32[1])], int32; + num_values=1) + chunk = metadata.row_groups[1].columns[1] + src = Parquet.source(bytes) + for offset in (typemax(UInt128), big(1) << 100, -(big(1) << 100)) + error = try + Parquet.readleafstream(src, chunk, schema.leaves[1], offset) + nothing + catch err + err + end + @test error isa ArgumentError + @test error.msg == "footer offset does not fit Int64" + end +end + +@testset "column chunk bounds and counts" begin + int32 = columnleaf(MD.Type.INT32) + page = datapage(Int32[1, 2, 3]) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=2) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=4) + @test_throws Parquet.FormatError readsynthetic([page, page], int32; num_values=3) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, total=length(page) + 1) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, total=length(page) - 1) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, total=-1) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=-1) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, data_page_offset=3) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, data_page_offset=5) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, data_page_offset=length(page) + 4) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, data_page_offset=typemax(Int64) - 1) + @test readsynthetic([page], int32; num_values=3, dictionary_page_offset=Int64(0)) == Int32[1, 2, 3] + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, dictionary_page_offset=Int64(4 + length(page) + 10)) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, dictionary_page_offset=Int64(2)) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, dictionary_page_offset=Int64(-1)) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=3, data_page_offset=-4) + # chunk range policy: the dictionary offset (if any) starts the chunk; data_page_offset is 0 or inside it + function chunkmeta(; data, index=nothing, dictionary=nothing, size=20) + return MD.ColumnMetaData(type_=MD.Type.INT32, encodings=[MD.Encoding.PLAIN], path_in_schema=["value"], + codec=MD.CompressionCodec.UNCOMPRESSED, num_values=Int64(0), total_uncompressed_size=Int64(size), + total_compressed_size=Int64(size), data_page_offset=Int64(data), + index_page_offset=index, dictionary_page_offset=dictionary) + end + @test Parquet._chunkrange(chunkmeta(; data=4), Int64(100)) == (4, 24) + @test Parquet._chunkrange(chunkmeta(; data=0, dictionary=Int64(4)), Int64(100)) == (4, 24) + @test Parquet._chunkrange(chunkmeta(; data=10, dictionary=Int64(4)), Int64(100)) == (4, 24) + @test_throws Parquet.FormatError Parquet._chunkrange( + chunkmeta(; data=24, dictionary=Int64(4)), Int64(100)) + @test Parquet._chunkrange(chunkmeta(; data=8, dictionary=Int64(0)), Int64(100)) == (8, 28) + @test Parquet._chunkrange(chunkmeta(; data=0, size=0), Int64(100)) == (0, 0) + @test Parquet._chunkrange(chunkmeta(; data=0, dictionary=Int64(0), + size=0), Int64(100)) == (0, 0) + @test_throws Parquet.FormatError Parquet._chunkrange( + chunkmeta(; data=1, size=0), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange( + chunkmeta(; data=4, size=0), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange( + chunkmeta(; data=0, dictionary=Int64(4), size=0), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange( + chunkmeta(; data=0, index=Int64(4), size=0), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=25, dictionary=Int64(4)), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=4, dictionary=Int64(8)), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=4, dictionary=Int64(-8)), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=-4), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=0), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=4), Int64(23)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=4), typemin(Int64)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=typemax(Int64)), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=4, size=typemax(Int64)), Int64(100)) + @test_throws Parquet.FormatError Parquet._chunkrange(chunkmeta(; data=typemax(Int64), dictionary=Int64(4)), Int64(100)) + # hostile footer offsets and sizes never widen the range past [4, footer) and never raise overflow errors + rng = MersenneTwister(77) + extremes = Int64[typemin(Int64), -1, 0, 1, 3, 4, 5, 99, 100, 101, typemax(Int64) - 1, typemax(Int64)] + for trial in 1:3000 + pick = () -> rand(rng, Bool) ? rand(rng, extremes) : rand(rng, Int64) + dictionary = rand(rng, Bool) ? nothing : pick() + md = chunkmeta(; data=pick(), dictionary=dictionary, size=pick()) + result = try + Parquet._chunkrange(md, Int64(100)) + catch err + err + end + if result isa Tuple + start, stop = result + @test start <= stop && (start == stop || (start >= 4 && stop <= 100)) + else + @test result isa Parquet.FormatError + end + end + bytes, meta, schema = syntheticfile([page], int32; num_values=3) + chunk = meta.row_groups[1].columns[1] + src = Parquet.source(bytes) + @test Parquet.readcolumn(src, chunk, schema.leaves[1], 4 + length(page)) == Int32[1, 2, 3] + @test_throws Parquet.FormatError Parquet.readcolumn(src, chunk, schema.leaves[1], 4 + length(page) - 1) + @test_throws Parquet.FormatError Parquet.readcolumn(src, chunk, schema.leaves[1], length(bytes) + 1) + file = Parquet.File(bytes) + @test_throws ArgumentError Parquet.readcolumn(file, meta, schema, 2, 1) + @test_throws ArgumentError Parquet.readcolumn(file, meta, schema, 1, 2) + @test_throws ArgumentError Parquet.readcolumn(file, meta, schema, 0, 1) + close(file) + _, wide, wideschema = syntheticfile([page], int32; num_values=3, extrachunks=1) + file = Parquet.File(bytes) + @test_throws Parquet.FormatError Parquet.readcolumn(file, wide, wideschema, 1, 1) + close(file) +end + +@testset "column chunk metadata validation" begin + int32 = columnleaf(MD.Type.INT32) + page = datapage(Int32[1]) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=1, file_path="other.parquet") + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=1, crypto=MD.ColumnCryptoMetaData(ENCRYPTION_WITH_FOOTER_KEY=MD.EncryptionWithFooterKey())) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=1, encryptedmeta=UInt8[0x01]) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=1, type=MD.Type.INT64) + @test_throws Parquet.FormatError readsynthetic([page], int32; num_values=1, path=["other"]) + @test_throws Parquet.UnsupportedFeatureError readsynthetic([page], + columnleaf(MD.Type.INT96); num_values=1) + repeated = columnleaf(MD.Type.INT32; repetition=MD.FieldRepetitionType.REPEATED) + @test_throws Parquet.FormatError readsynthetic([page], repeated; num_values=1) + bytes, meta, schema = syntheticfile([page], int32; num_values=1) + nometa = MD.ColumnChunk() + @test_throws Parquet.FormatError Parquet.readcolumn(Parquet.source(bytes), nometa, schema.leaves[1], 4 + length(page)) + @test_throws Parquet.LimitError readsynthetic([page], int32; num_values=1, limits=Parquet.Limits(max_container_elements=0)) + @test_throws Parquet.LimitError readsynthetic([page], int32; num_values=1, limits=Parquet.Limits(max_page_bytes=3)) + @test_throws Parquet.LimitError readsynthetic([page], int32; num_values=1, limits=Parquet.Limits(max_page_header_bytes=2)) + @test readsynthetic([page], int32; num_values=1, limits=Parquet.Limits(max_page_bytes=4)) == Int32[1] +end + +@testset "column mutation fuzz" begin + int32 = columnleaf(MD.Type.INT32; repetition=MD.FieldRepetitionType.OPTIONAL) + pages = [datapage(Int32[1, 2]; levels=[1, 0, 1], maxlevel=1), datapage(Int32[3]; levels=[1, 0], maxlevel=1)] + bytes, meta, schema = syntheticfile(pages, int32; num_values=5) + region = 5:(4 + sum(length, pages)) + rng = MersenneTwister(4242) + outcomes = Set{Symbol}() + for trial in 1:800 + mutated = copy(bytes) + for _ in 1:rand(rng, 1:3) + mutated[rand(rng, region)] = rand(rng, UInt8) + end + file = Parquet.File(mutated) + result = try + Parquet.readcolumn(file, meta, schema, 1, 1) + :ok + catch err + err + end + close(file) + if result === :ok + push!(outcomes, :ok) + else + @test result isa Union{Parquet.FormatError,Parquet.LimitError} + push!(outcomes, nameof(typeof(result))) + end + end + @test :FormatError in outcomes +end + +function corpuscolumn(path::String, column::Int) + file = Parquet.File(path) + meta = TH.decode(file.footer.bytes, MD.FileMetaData) + schema = Parquet.Schema(meta) + values = Parquet.readcolumn(file, meta, schema, 1, column) + close(file) + return values, meta +end + +# Row-weighted digest used to compare against the pyarrow oracle (see the handoff notes). +function oracledigest(values) + digest = UInt64(0) + nulls = Int[] + for (index, value) in enumerate(values) + if value === missing + push!(nulls, index) + continue + end + word = value isa Vector{UInt8} ? foldl((acc, byte) -> (acc << 8) | UInt64(byte), value; init=UInt64(0)) : + value isa Bool ? UInt64(value) : reinterpret(UInt64, Int64(value)) + digest += UInt64(index) * word + end + return digest, nulls +end + +@testset "official flat PLAIN fixtures" begin + if !isdir(columncorpus()) + @warn "parquet-testing corpus not found; skipping column corpus tests" COLUMN_CORPUS + else + a, meta = corpuscolumn(columncorpus("datapage_v1-uncompressed-checksum.parquet"), 1) + b, _ = corpuscolumn(columncorpus("datapage_v1-uncompressed-checksum.parquet"), 2) + @test a isa Vector{Int32} && length(a) == 5120 && sum(Int64, a) == 43118090240 && oracledigest(a)[1] == 454385326080 + @test a[1:4] == Int32[50462976, 117835012, 185207048, 252579084] && a[(end - 1):end] == Int32[84281096, 16909060] + @test extrema(a) == (-2122153084, 2138996092) + @test b isa Vector{Int32} && length(b) == 5120 && sum(Int64, b) == 129016125440 && oracledigest(b)[1] == 378853655132160 + @test b[1:2] == Int32[1734763876, 1802135912] && b[end] == -1684366952 && extrema(b) == (-2088599168, 2138996092) + chunks = meta.row_groups[1].columns + @test [chunk.meta_data.encodings for chunk in chunks] == [[MD.Encoding.RLE, MD.Encoding.PLAIN], [MD.Encoding.RLE, MD.Encoding.PLAIN]] + @test [chunk.meta_data.data_page_offset for chunk in chunks] == [4, 20540] + @test all(chunk -> chunk.meta_data.dictionary_page_offset === nothing && chunk.meta_data.statistics === nothing, chunks) + @test all(chunk -> chunk.meta_data.encoding_stats == [MD.PageEncodingStats(page_type=MD.PageType.DATA_PAGE, encoding=MD.Encoding.PLAIN, count=Int32(2))], chunks) + @test startswith(meta.created_by, "parquet-mr version 1.13.0-SNAPSHOT") + @test_throws Parquet.FormatError corpuscolumn(columncorpus("datapage_v1-corrupt-checksum.parquet"), 1) + @test_throws Parquet.FormatError corpuscolumn(columncorpus("datapage_v1-corrupt-checksum.parquet"), 2) + nullable, nullmeta = corpuscolumn(columncorpus("int32_with_null_pages.parquet"), 1) + digest, nulls = oracledigest(nullable) + @test nullable isa Vector{Union{Missing,Int32}} && length(nullable) == 1000 && length(nulls) == 275 + @test digest == 18446743417863037648 && sum(nulls) == 92306 && nulls[1:10] == [5, 14, 34, 47, 57, 67, 80, 83, 102, 116] + @test sum(Int64, skipmissing(nullable)) == -12383254597 + @test isequal(nullable[1:5], Union{Missing,Int32}[-654807448, -465559769, -34563097, 398454479, missing]) + @test [count(ismissing, nullable[((page - 1) * 100 + 1):(page * 100)]) for page in 1:10] == [8, 55, 100, 52, 16, 12, 5, 7, 8, 12] + @test nullmeta.row_groups[1].columns[1].meta_data.statistics.null_count == 275 + fixed, fixedmeta = corpuscolumn(columncorpus("fixed_length_byte_array.parquet"), 1) + digest, nulls = oracledigest(fixed) + @test fixed isa Vector{Union{Missing,Vector{UInt8}}} && length(fixed) == 1000 && length(nulls) == 105 + @test digest == 148827896 && sum(nulls) == 43965 + integers = [foldl((acc, byte) -> (acc << 8) | Int(byte), value; init=0) for value in skipmissing(fixed)] + @test integers[1:6] == [1000, 990, 989, 988, 987, 986] && integers[(end - 2):end] == [3, 2, 1] && sum(integers) == 439360 + @test all(integers[i] > integers[i + 1] for i in 1:(length(integers) - 1)) + @test all(value -> value === missing || length(value) == 4, fixed) + @test [count(ismissing, fixed[((page - 1) * 100 + 1):(page * 100)]) for page in 1:10] == [9, 9, 19, 10, 13, 11, 11, 8, 9, 6] + bson, _ = corpuscolumn(columncorpus("bson.parquet"), 1) + @test isequal(bson, Union{Missing,Vector{UInt8}}[hex2bytes("0c0000001061000100000000"), hex2bytes("0f000000106100010000000a620000"), missing]) + json, _ = corpuscolumn(columncorpus("json.parquet"), 1) + @test isequal(json, Union{Missing,Vector{UInt8}}[collect(codeunits("{\"a\":1}")), collect(codeunits("{\"a\":1,\"b\":null}")), collect(codeunits("[1,null,3]")), missing]) + binary, _ = corpuscolumn(columncorpus("binary.parquet"), 1) + @test isequal(binary, Union{Missing,Vector{UInt8}}[[UInt8(i)] for i in 0:11]) + bools, _ = corpuscolumn(columncorpus("alltypes_plain.parquet"), 2) + @test isequal(bools, Union{Missing,Bool}[true, false, true, false, true, false, true, false]) + floats, _ = corpuscolumn(columncorpus("floating_orders_nan_count.parquet"), 1) + @test floats isa Vector{Float32} && length(floats) == 10 + # parquet-cpp-arrow 17 writes data_page_offset = 0 for chunks that hold only a dictionary page + emptyfile = Parquet.File(columncorpus("column_chunk_key_value_metadata.parquet")) + emptymeta = TH.decode(emptyfile.footer.bytes, MD.FileMetaData) + emptychunks = emptymeta.row_groups[1].columns + @test emptymeta.num_rows == 0 && [chunk.meta_data.data_page_offset for chunk in emptychunks] == [0, 0] + @test [Parquet._chunkrange(chunk.meta_data, emptyfile.footer.offset) for chunk in emptychunks] == [(4, 18), (97, 111)] + @test Parquet.readcolumn(emptyfile, emptymeta, Parquet.Schema(emptymeta), 1, 1) == Int32[] + close(emptyfile) + # seeded mutations inside the first column chunk of the checksum fixture + path = columncorpus("datapage_v1-uncompressed-checksum.parquet") + original = read(path) + file = Parquet.File(path) + meta = TH.decode(file.footer.bytes, MD.FileMetaData) + close(file) + schema = Parquet.Schema(meta) + md = meta.row_groups[1].columns[1].meta_data + region = (md.data_page_offset + 1):(md.data_page_offset + md.total_compressed_size) + rng = MersenneTwister(99) + for trial in 1:200 + mutated = copy(original) + for _ in 1:rand(rng, 1:2) + mutated[rand(rng, region)] = rand(rng, UInt8) + end + mutatedfile = Parquet.File(mutated) + result = try + Parquet.readcolumn(mutatedfile, meta, schema, 1, 1) + :ok + catch err + err + end + close(mutatedfile) + @test result === :ok || result isa Union{Parquet.FormatError, + Parquet.LimitError} + end + end +end + +@testset "official V2 and encoded flat fixtures" begin + if !isdir(columncorpus()) + @warn "parquet-testing corpus not found; skipping V2 column corpus tests" COLUMN_CORPUS + else + gzip = Parquet.Table(columncorpus("concatenated_gzip_members.parquet")) + @test gzip.columns.long_col == collect(Int64, 1:513) + close(gzip) + + booleans = Parquet.Table(columncorpus("rle_boolean_encoding.parquet")) + booleanvalues = booleans.columns.datatype_boolean + @test length(booleanvalues) == 68 + @test findall(ismissing, booleanvalues) == [3, 16, 24, 39, 49, 61] + @test isequal(booleanvalues[1:8], Union{Missing,Bool}[true, false, missing, true, + true, false, false, true]) + close(booleans) + + for fixture in ("page_v2_empty_compressed.parquet", + "datapage_v2_empty_datapage.snappy.parquet") + empty = Parquet.Table(columncorpus(fixture)) + @test all(ismissing, first(values(empty.columns))) + close(empty) + end + + required = Parquet.Table(columncorpus("delta_encoding_required_column.parquet")) + firstrequired = first(values(required.columns)) + @test firstrequired[1:5] == Int32[105, 104, 103, 102, 101] + @test firstrequired[(end - 2):end] == Int32[3, 2, 1] + close(required) + optional = Parquet.Table(columncorpus("delta_encoding_optional_column.parquet")) + firstoptional = first(values(optional.columns)) + @test isequal(firstoptional[1:5], Union{Missing,Int64}[100, 99, 98, 97, 96]) + @test isequal(firstoptional[(end - 2):end], Union{Missing,Int64}[3, 2, 1]) + close(optional) + + delta = Parquet.Table(columncorpus("delta_byte_array.parquet")) + @test delta.columns.c_customer_id[1:3] == + ["AAAAAAAAIODAAAAA", "AAAAAAAAHODAAAAA", "AAAAAAAAGODAAAAA"] + @test delta.columns.c_customer_id[(end - 2):end] == + ["AAAAAAAADAAAAAAA", "AAAAAAAACAAAAAAA", "AAAAAAAABAAAAAAA"] + close(delta) + lengths = Parquet.Table(columncorpus("delta_length_byte_array.parquet")) + @test lengths.columns.FRUIT == + ["apple_banana_mango$((index - 1)^2)" for index in 1:1000] + close(lengths) + + packed = Parquet.Table(columncorpus("delta_binary_packed.parquet")) + @test all(==(Int64(6374628540732951412)), packed.columns.bitwidth0) + @test packed.columns.bitwidth1[1:5] == Int64[0, -1, -1, -1, -1] + @test packed.columns.bitwidth1[(end - 2):end] == Int64[-102, -103, -104] + close(packed) + + split = Parquet.Table(columncorpus("byte_stream_split_extended.gzip.parquet")) + for (plain, encoded) in ((:float_plain, :float_byte_stream_split), + (:double_plain, :double_byte_stream_split), + (:int32_plain, :int32_byte_stream_split), + (:int64_plain, :int64_byte_stream_split), + (:flba5_plain, :flba5_byte_stream_split)) + @test isequal(getproperty(split.columns, plain), getproperty(split.columns, encoded)) + end + close(split) + + path = columncorpus("datapage_v2.snappy.parquet") + file = Parquet.File(path) + metadata = TH.decode(copy(file.footer.bytes), MD.FileMetaData) + schema = Parquet.Schema(metadata) + abc = collect(codeunits("abc")) + @test isequal(Parquet.readcolumn(file, metadata, schema, 1, 1), + Union{Missing,Vector{UInt8}}[abc, abc, abc, missing, abc]) + @test Parquet.readcolumn(file, metadata, schema, 1, 2) == Int32[1, 2, 3, 4, 5] + @test Parquet.readcolumn(file, metadata, schema, 1, 3) == [2.0, 3.0, 4.0, 5.0, 2.0] + @test Parquet.readcolumn(file, metadata, schema, 1, 4) == Bool[1, 1, 1, 0, 1] + close(file) + end +end diff --git a/test/conformance/features.toml b/test/conformance/features.toml new file mode 100644 index 0000000..f56423f --- /dev/null +++ b/test/conformance/features.toml @@ -0,0 +1,109 @@ +format_version = "2.13.0" +format_commit = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" +corpus_commit = "09f3cdbde45302f0f0c689c950e465e98a9df960" +policy = "complete means read, write when applicable, malformed-input, and independent-oracle evidence" + +[[features]] +name = "compact_protocol_metadata" +stage = 1 +status = "in_progress" + +[[features]] +name = "plain_encoding" +stage = 2 +status = "in_progress" + +[[features]] +name = "rle_and_bit_packed_hybrid" +stage = 2 +status = "in_progress" + +[[features]] +name = "schema_tree_and_levels" +stage = 2 +status = "in_progress" + +[[features]] +name = "data_page_v1" +stage = 2 +status = "in_progress" + +[[features]] +name = "page_crc32" +stage = 2 +status = "in_progress" + +[[features]] +name = "data_page_v2" +stage = 3 +status = "in_progress" + +[[features]] +name = "dictionary_encoding" +stage = 3 +status = "in_progress" + +[[features]] +name = "delta_encodings" +stage = 3 +status = "in_progress" + +[[features]] +name = "byte_stream_split" +stage = 3 +status = "in_progress" + +[[features]] +name = "compression_codecs" +stage = 3 +status = "in_progress" + +[[features]] +name = "logical_types" +stage = 4 +status = "in_progress" + +[[features]] +name = "nested_data" +stage = 4 +status = "in_progress" + +[[features]] +name = "statistics_and_column_order" +stage = 5 +status = "planned" + +[[features]] +name = "column_and_offset_indexes" +stage = 5 +status = "planned" + +[[features]] +name = "split_block_bloom_filter" +stage = 5 +status = "planned" + +[[features]] +name = "scan_pushdown" +stage = 5 +status = "planned" + +[[features]] +name = "encryption" +stage = 6 +status = "planned" + +[[features]] +name = "variant" +stage = 7 +status = "planned" + +[[features]] +name = "geospatial" +stage = 7 +status = "planned" + +[[features]] +name = "dataset" +stage = 8 +status = "planned" diff --git a/test/conformance/n5/.gitignore b/test/conformance/n5/.gitignore new file mode 100644 index 0000000..62d48db --- /dev/null +++ b/test/conformance/n5/.gitignore @@ -0,0 +1,2 @@ +output/ +*.oci.tar diff --git a/test/conformance/n5/bootstrap-oracles.sh b/test/conformance/n5/bootstrap-oracles.sh new file mode 100755 index 0000000..cbd6c8b --- /dev/null +++ b/test/conformance/n5/bootstrap-oracles.sh @@ -0,0 +1,232 @@ +#!/bin/sh +set -eu + +N5_REPOSITORY=ghcr.io/juliaio/parquet-jl-n5-oracles + +usage() { + echo "usage: bootstrap-oracles.sh --output LOCK [--publish $N5_REPOSITORY]" >&2 + exit 64 +} + +fail() { + echo "bootstrap-oracles.sh: $1" >&2 + exit 65 +} + +is_sha256() { + n5_value=$1 + case "$n5_value" in + sha256:*) n5_hex=${n5_value#sha256:} ;; + *) return 1 ;; + esac + [ "${#n5_hex}" -eq 64 ] || return 1 + case "$n5_hex" in + *[!0-9a-f]*) return 1 ;; + esac + return 0 +} + +host_sha256() { + n5_file=$1 + [ -f "$n5_file" ] || fail "required file $n5_file is absent" + if command -v sha256sum >/dev/null 2>&1; then + n5_output=$(sha256sum "$n5_file") || fail "cannot hash $n5_file" + else + n5_output=$(shasum -a 256 "$n5_file") || fail "cannot hash $n5_file" + fi + n5_hash=${n5_output%% *} + is_sha256 "sha256:$n5_hash" || fail "invalid SHA-256 output for $n5_file" + printf '%s\n' "$n5_hash" +} + +image_sha256() { + n5_file=$1 + n5_output=$(docker run --rm --platform linux/amd64 --network none \ + --entrypoint sha256sum "$n5_image" "$n5_file") || + fail "cannot hash $n5_file in $n5_image" + n5_hash=${n5_output%% *} + is_sha256 "sha256:$n5_hash" || fail "invalid image SHA-256 for $n5_file" + printf '%s\n' "$n5_hash" +} + +verify_repo_digest() { + n5_repo_digests=$(docker image inspect "$n5_image" \ + --format '{{join .RepoDigests "\n"}}') || + fail "cannot inspect $n5_image" + n5_matched=0 + for n5_candidate in $n5_repo_digests; do + [ "$n5_candidate" = "$n5_image" ] && n5_matched=1 + done + [ "$n5_matched" -eq 1 ] || fail "pulled RepoDigest does not match $n5_image" +} + +n5_root=$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd) +n5_oracle_dir="$n5_root/test/conformance/n5/oracles" +n5_output= +n5_repository= +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + [ "$#" -ge 2 ] || usage + n5_output=$2 + shift 2 + ;; + --publish) + [ "$#" -ge 2 ] || usage + n5_repository=$2 + shift 2 + ;; + *) + usage + ;; + esac +done +[ -n "$n5_output" ] || usage +if [ -n "$n5_repository" ] && [ "$n5_repository" != "$N5_REPOSITORY" ]; then + fail "--publish accepts only $N5_REPOSITORY" +fi +command -v docker >/dev/null 2>&1 || { + echo "bootstrap-oracles.sh: Docker is required" >&2 + exit 69 +} +n5_output_dir=$(dirname -- "$n5_output") +[ -d "$n5_output_dir" ] || fail "lock output directory is absent" +n5_fixture_path="$n5_root/test/conformance/n5/manifest.toml" +n5_fixture_manifest_before=$(host_sha256 "$n5_fixture_path") +n5_verified_existing=0 + +n5_work=$(mktemp -d) +n5_staged_lock= +n5_local_tag= +n5_published_tag= +cleanup() { + rm -rf "$n5_work" + [ -z "$n5_staged_lock" ] || rm -f "$n5_staged_lock" + [ -z "$n5_local_tag" ] || + docker image rm "$n5_local_tag" >/dev/null 2>&1 || true + [ -z "$n5_published_tag" ] || + docker image rm "$n5_published_tag" >/dev/null 2>&1 || true +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +n5_local_tag="parquet-jl-n5-oracles:bootstrap-$$" +n5_image_archive="$n5_work/oracles.oci.tar" +docker buildx build \ + --no-cache \ + --file "$n5_oracle_dir/Dockerfile" \ + --build-arg SOURCE_DATE_EPOCH=1787356800 \ + --output "type=oci,dest=$n5_image_archive,rewrite-timestamp=true" \ + --platform linux/amd64 \ + --provenance=false \ + --sbom=false \ + --tag "$n5_local_tag" \ + "$n5_oracle_dir" +docker load --input "$n5_image_archive" >/dev/null +rm "$n5_image_archive" +docker run --rm --platform linux/amd64 --network none "$n5_local_tag" +n5_image_id=$(docker image inspect "$n5_local_tag" --format '{{.Id}}') || + fail "cannot inspect the validated local image" +is_sha256 "$n5_image_id" || fail "validated local image has no exact image ID" +if [ -f "$n5_output" ]; then + n5_expected_image=$(sed -n \ + 's/^image_reference[[:space:]]*=[[:space:]]*"\([^"]*\)"[[:space:]]*$/\1/p' \ + "$n5_output") || fail "cannot inspect existing oracle lock" + case "$n5_expected_image" in + "$N5_REPOSITORY"@sha256:*) ;; + *) fail "existing oracle lock has no exact image reference" ;; + esac + docker image inspect "$n5_expected_image" >/dev/null 2>&1 || + docker pull --platform linux/amd64 "$n5_expected_image" >/dev/null + n5_expected_id=$(docker image inspect "$n5_expected_image" --format '{{.Id}}') || + fail "cannot inspect existing locked oracle image" + [ "$n5_image_id" = "$n5_expected_id" ] || + fail "clean build is not reproducible with the existing oracle lock" + n5_verified_existing=1 +fi +if [ -z "$n5_repository" ]; then + if [ "$n5_verified_existing" -eq 1 ]; then + echo "bootstrap-oracles.sh: clean image $n5_image_id matches the existing lock" >&2 + exit 0 + fi + echo "bootstrap-oracles.sh: local image $n5_image_id passed the offline check" >&2 + echo "bootstrap-oracles.sh: no lock was written because no published repository digest exists" >&2 + echo "bootstrap-oracles.sh: rerun with explicit --publish $N5_REPOSITORY after publication is authorized" >&2 + exit 2 +fi + +n5_image_hex=${n5_image_id#sha256:} +n5_published_tag="$n5_repository:bootstrap-$n5_image_hex" +docker image tag "$n5_local_tag" "$n5_published_tag" +n5_push_output=$(docker image push --quiet --platform linux/amd64 \ + "$n5_published_tag" 2>&1) || fail "publishing the validated image failed" +n5_digest= +for n5_word in $n5_push_output; do + if is_sha256 "$n5_word"; then + n5_digest=$n5_word + fi +done +if [ -z "$n5_digest" ]; then + n5_repo_digests=$(docker image inspect "$n5_published_tag" \ + --format '{{join .RepoDigests "\n"}}') || + fail "cannot inspect the published image" + for n5_candidate in $n5_repo_digests; do + case "$n5_candidate" in + "$n5_repository"@sha256:*) n5_digest=${n5_candidate#*@} ;; + esac + done +fi +is_sha256 "$n5_digest" || fail "published image digest is absent" +n5_image="$n5_repository@$n5_digest" +docker pull --platform linux/amd64 "$n5_image" >/dev/null +verify_repo_digest +n5_pulled_id=$(docker image inspect "$n5_image" --format '{{.Id}}') || + fail "cannot inspect the pulled image" +[ "$n5_pulled_id" = "$n5_image_id" ] || + fail "published image differs from the validated local image" +n5_platform=$(docker image inspect "$n5_image" \ + --format '{{.Os}}/{{.Architecture}}') || fail "cannot inspect image platform" +[ "$n5_platform" = "linux/amd64" ] || fail "published image is not linux/amd64" +docker run --rm --platform linux/amd64 --network none "$n5_image" + +n5_maven_manifest=$(image_sha256 /opt/n5/manifests/maven-artifacts.sha256) +n5_cargo_manifest=$(image_sha256 /opt/n5/manifests/cargo-vendor.sha256) +n5_source_manifest=$(image_sha256 /opt/n5/manifests/harness-source.sha256) +n5_maven_tree=$(image_sha256 /opt/n5/manifests/maven-dependency-tree.txt) +n5_cargo_tree=$(image_sha256 /opt/n5/manifests/cargo-dependency-tree.txt) +n5_fixture_manifest=$(host_sha256 "$n5_fixture_path") +[ "$n5_fixture_manifest" = "$n5_fixture_manifest_before" ] || + fail "fixture manifest changed during bootstrap" + +n5_lock="$n5_work/oracles.lock" +{ + printf 'schema_version = 1\n' + printf 'platform = "linux/amd64"\n' + printf 'image_repository = "%s"\n' "$N5_REPOSITORY" + printf 'image_digest = "%s"\n' "$n5_digest" + printf 'image_reference = "%s"\n' "$n5_image" + printf 'base_image_digest = "sha256:ab2527b3c9b7c15bc88f60dec19b2aa39939a6e0045fb8f538eeecbd7af59c69"\n' + printf 'maven_version = "3.9.8"\n' + printf 'maven_archive_sha512 = "7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2"\n' + printf 'maven_artifacts_manifest_sha256 = "%s"\n' "$n5_maven_manifest" + printf 'maven_dependency_tree_sha256 = "%s"\n' "$n5_maven_tree" + printf 'rust_version = "1.96.1"\n' + printf 'rust_channel_manifest_sha256 = "87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975"\n' + printf 'rust_tarball_sha256 = "d29ccb1559a177c4e72291f6e5f629de7fe8885e7521ca47802627544b121e95"\n' + printf 'cargo_vendor_manifest_sha256 = "%s"\n' "$n5_cargo_manifest" + printf 'cargo_dependency_tree_sha256 = "%s"\n' "$n5_cargo_tree" + printf 'harness_source_manifest_sha256 = "%s"\n' "$n5_source_manifest" + printf 'parquet_java_version = "1.17.1"\n' + printf 'parquet_java_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81"\n' + printf 'arrow_rs_version = "59.2.0"\n' + printf 'arrow_rs_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a"\n' + printf 'parquet_testing_commit = "09f3cdbde45302f0f0c689c950e465e98a9df960"\n' + printf 'fixture_manifest_sha256 = "%s"\n' "$n5_fixture_manifest" +} > "$n5_lock" +n5_staged_lock=$(mktemp "$n5_output_dir/.n5-oracles.lock.XXXXXX") +install -m 0444 "$n5_lock" "$n5_staged_lock" +mv -f "$n5_staged_lock" "$n5_output" +n5_staged_lock= +echo "wrote $n5_output for $n5_image" diff --git a/test/conformance/n5/compare-evidence.pl b/test/conformance/n5/compare-evidence.pl new file mode 100644 index 0000000..ab272ee --- /dev/null +++ b/test/conformance/n5/compare-evidence.pl @@ -0,0 +1,812 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use JSON::PP; + +sub fail { + die "compare-evidence.pl: $_[0]\n"; +} + +sub usage { + die "usage: compare-evidence.pl --manifest FILE --unsupported FILE " + . "--java FILE --rust FILE --output FILE\n"; +} + +sub arguments { + my %values; + while (@ARGV) { + my $key = shift @ARGV; + usage() unless $key =~ /^--(?:manifest|unsupported|java|rust|output)$/; + usage() unless @ARGV; + fail("duplicate option $key") if exists $values{$key}; + $values{$key} = shift @ARGV; + } + for my $key (qw(--manifest --unsupported --java --rust --output)) { + usage() unless exists $values{$key}; + } + return %values; +} + +sub lines { + my ($path) = @_; + open my $input, '<:raw', $path or fail("cannot open $path: $!"); + my @lines = <$input>; + close $input or fail("cannot close $path: $!"); + chomp @lines; + s/\r$// for @lines; + return @lines; +} + +sub normalized_path { + my ($path, $label) = @_; + fail("$label is empty") unless length $path; + fail("$label is absolute") if $path =~ m{^/}; + fail("$label contains a backslash") if $path =~ /\\/; + fail("$label is not normalized") if grep { $_ eq '' || $_ eq '.' || $_ eq '..' } + split m{/}, $path, -1; + return $path; +} + +sub parse_manifest { + my ($path) = @_; + my @input = lines($path); + fail("fixture manifest is empty") unless @input; + my $header = shift @input; + my $expected = join "\t", qw(kind case_id page_version codec reference + target reference_sha256 target_sha256); + fail("fixture manifest header differs") unless $header eq $expected; + my @rows; + my %targets; + for my $index (0 .. $#input) { + my @fields = split /\t/, $input[$index], -1; + fail("fixture manifest row @{[$index + 2]} has wrong field count") + unless @fields == 8; + my ($kind, $case, $page, $codec, $reference, $target, + $reference_hash, $target_hash) = @fields; + fail("unsupported fixture kind $kind") + unless $kind =~ /^(?:binding|provenance|property|external)$/; + fail("empty case ID") unless length $case; + fail("unsupported page version $page") unless $page =~ /^v[12]$/; + fail("unsupported codec $codec") unless $codec =~ + /^(?:uncompressed|snappy|gzip|brotli|zstd|lz4_raw)$/; + normalized_path($reference, "reference path"); + normalized_path($target, "target path"); + fail("invalid reference SHA-256") unless $reference_hash =~ /^[0-9a-f]{64}$/; + fail("invalid target SHA-256") unless $target_hash =~ /^[0-9a-f]{64}$/; + fail("duplicate target $target") if $targets{$target}++; + push @rows, { + kind => $kind, + case_id => $case, + page_version => $page, + codec => $codec, + reference => $reference, + target => $target, + reference_sha256 => $reference_hash, + target_sha256 => $target_hash, + }; + } + fail("fixture manifest must contain 256 mappings") unless @rows == 256; + return @rows; +} + +sub parse_unsupported { + my ($path) = @_; + my @input = lines($path); + fail("unsupported allowlist is empty") unless @input; + fail("unsupported allowlist header differs") + unless shift(@input) eq join("\t", qw(oracle file error_class error_message)); + my %expected; + for my $index (0 .. $#input) { + my @fields = split /\t/, $input[$index], -1; + fail("unsupported row @{[$index + 2]} has wrong field count") + unless @fields == 4; + my ($oracle, $file, $class, $message) = @fields; + fail("unknown unsupported oracle $oracle") + unless $oracle eq 'parquet-java' || $oracle eq 'arrow-rs'; + normalized_path($file, "unsupported file"); + fail("empty unsupported class") unless length $class; + fail("empty unsupported message") unless length $message; + my $key = "$oracle\0$file"; + fail("duplicate unsupported key $oracle/$file") if exists $expected{$key}; + $expected{$key} = [$class, $message]; + } + return %expected; +} + +sub parse_json_value { + my ($text, $label) = @_; + my $value = eval { JSON::PP->new->utf8->decode($text) }; + fail("cannot decode $label: $@") if $@; + return $value; +} + +sub exact_keys { + my ($value, $required, $label) = @_; + fail("$label is not an object") unless ref($value) eq 'HASH'; + my %expected = map { $_ => 1 } @$required; + my @actual = sort keys %$value; + my @wanted = sort keys %expected; + fail("$label keys differ") unless join("\0", @actual) eq join("\0", @wanted); +} + +sub require_object { + my ($value, $label) = @_; + fail("$label is not an object") unless ref($value) eq 'HASH'; + return $value; +} + +sub require_array { + my ($value, $label) = @_; + fail("$label is not an array") unless ref($value) eq 'ARRAY'; + return $value; +} + +sub require_string { + my ($value, $label) = @_; + fail("$label is not a string") unless defined($value) && !ref($value) + && JSON::PP->new->allow_nonref->encode($value) =~ /^"/; + return $value; +} + +sub require_integer { + my ($value, $label) = @_; + fail("$label is not an integer") unless defined($value) && !ref($value) + && JSON::PP->new->allow_nonref->encode($value) =~ + /^-?(?:0|[1-9][0-9]*)$/; + return 0 + $value; +} + +sub require_nonnegative_integer { + my ($value, $label) = @_; + my $integer = require_integer($value, $label); + fail("$label is negative") if $integer < 0; + return $integer; +} + +sub require_false { + my ($value, $label) = @_; + fail("$label is not false") unless JSON::PP::is_bool($value) && !$value; +} + +sub validate_java_avro { + my ($value, $label) = @_; + exact_keys($value, [qw(status read_schema materialized_schema rows + normalized_rows error_class error_message error_stack exception_chain)], $label); + require_string($value->{status}, "$label status"); + require_array($value->{rows}, "$label rows"); + require_array($value->{normalized_rows}, "$label normalized rows"); + require_array($value->{error_stack}, "$label error stack"); + require_array($value->{exception_chain}, "$label exception chain"); +} + +sub validate_java_file_record { + my ($record, $label) = @_; + exact_keys($record, [qw(record file sha256 case_id page_version created_by + row_count metadata physical_schema row_groups raw_group_rows columns avro)], $label); + require_string($record->{file}, "$label file"); + fail("$label SHA-256 is invalid") unless + require_string($record->{sha256}, "$label SHA-256") =~ /^[0-9a-f]{64}$/; + require_nonnegative_integer($record->{row_count}, "$label row count"); + require_object($record->{metadata}, "$label metadata"); + require_string($record->{physical_schema}, "$label physical schema"); + my $row_groups = require_array($record->{row_groups}, "$label row groups"); + require_array($record->{raw_group_rows}, "$label raw rows"); + my $columns = require_array($record->{columns}, "$label columns"); + for my $index (0 .. $#$row_groups) { + my $group = $row_groups->[$index]; + exact_keys($group, [qw(ordinal row_count total_byte_size columns)], + "$label row group $index"); + require_nonnegative_integer($group->{ordinal}, "$label row group ordinal"); + require_nonnegative_integer($group->{row_count}, "$label row group rows"); + require_nonnegative_integer($group->{total_byte_size}, "$label row group bytes"); + my $chunks = require_array($group->{columns}, "$label row group columns"); + for my $chunk_index (0 .. $#$chunks) { + my $chunk = $chunks->[$chunk_index]; + exact_keys($chunk, [qw(path value_count codec encodings + total_compressed_size total_uncompressed_size)], + "$label row group $index chunk $chunk_index"); + require_string($chunk->{path}, "$label chunk path"); + require_string($chunk->{codec}, "$label chunk codec"); + require_array($chunk->{encodings}, "$label chunk encodings"); + require_nonnegative_integer($chunk->{value_count}, "$label chunk values"); + require_nonnegative_integer($chunk->{total_compressed_size}, + "$label chunk compressed bytes"); + require_nonnegative_integer($chunk->{total_uncompressed_size}, + "$label chunk uncompressed bytes"); + } + } + for my $index (0 .. $#$columns) { + my $column = $columns->[$index]; + exact_keys($column, [qw(path physical_type logical_type type_length + max_repetition_level max_definition_level repetition definition dense + dictionaries pages row_groups)], "$label column $index"); + require_string($column->{path}, "$label column path"); + require_string($column->{physical_type}, "$label column physical type"); + require_integer($column->{type_length}, "$label column type length"); + require_nonnegative_integer($column->{max_repetition_level}, + "$label column maximum repetition level"); + require_nonnegative_integer($column->{max_definition_level}, + "$label column maximum definition level"); + require_array($column->{repetition}, "$label column repetitions"); + require_array($column->{definition}, "$label column definitions"); + require_array($column->{dense}, "$label column dense values"); + require_array($column->{dictionaries}, "$label column dictionaries"); + my $pages = require_array($column->{pages}, "$label column pages"); + for my $page_index (0 .. $#$pages) { + my $page = $pages->[$page_index]; + exact_keys($page, [qw(row_group ordinal type encoding value_count + row_count null_count compressed_size uncompressed_size + index_row_count)], "$label column $index page $page_index"); + require_string($page->{type}, "$label page type"); + require_string($page->{encoding}, "$label page encoding"); + for my $field (qw(row_group ordinal value_count row_count null_count + compressed_size uncompressed_size index_row_count)) { + require_nonnegative_integer($page->{$field}, "$label page $field") + if defined $page->{$field}; + } + } + require_array($column->{row_groups}, "$label column row groups"); + } + my $avro = require_object($record->{avro}, "$label Avro"); + exact_keys($avro, [qw(add_list_element_records inferred explicit)], "$label Avro"); + require_false($avro->{add_list_element_records}, "$label Avro list setting"); + validate_java_avro($avro->{inferred}, "$label inferred Avro"); + validate_java_avro($avro->{explicit}, "$label explicit Avro") + if defined $avro->{explicit}; +} + +sub validate_rust_file_evidence { + my ($value, $label) = @_; + exact_keys($value, [qw(case_id file_name sha256 file_bytes rows row_groups + physical_schema columns arrow)], $label); + require_string($value->{case_id}, "$label case ID"); + require_string($value->{file_name}, "$label file name"); + fail("$label SHA-256 is invalid") unless + require_string($value->{sha256}, "$label SHA-256") =~ /^[0-9a-f]{64}$/; + require_nonnegative_integer($value->{file_bytes}, "$label file bytes"); + require_nonnegative_integer($value->{rows}, "$label rows"); + require_nonnegative_integer($value->{row_groups}, "$label row-group count"); + require_string($value->{physical_schema}, "$label physical schema"); + my $columns = require_array($value->{columns}, "$label columns"); + for my $index (0 .. $#$columns) { + my $column = $columns->[$index]; + exact_keys($column, [qw(path physical_type maximum_definition_level + maximum_repetition_level row_groups)], "$label column $index"); + require_array($column->{path}, "$label column path"); + require_string($column->{physical_type}, "$label column physical type"); + require_nonnegative_integer($column->{maximum_definition_level}, + "$label column maximum definition level"); + require_nonnegative_integer($column->{maximum_repetition_level}, + "$label column maximum repetition level"); + my $groups = require_array($column->{row_groups}, "$label column row groups"); + for my $group_index (0 .. $#$groups) { + my $group = $groups->[$group_index]; + exact_keys($group, [qw(row_group rows compression repetition definition + dense_values pages)], "$label column $index row group $group_index"); + require_nonnegative_integer($group->{row_group}, "$label row-group ordinal"); + require_nonnegative_integer($group->{rows}, "$label row-group rows"); + require_string($group->{compression}, "$label row-group compression"); + require_array($group->{repetition}, "$label row-group repetitions"); + require_array($group->{definition}, "$label row-group definitions"); + require_array($group->{dense_values}, "$label row-group dense values"); + my $pages = require_array($group->{pages}, "$label row-group pages"); + for my $page_index (0 .. $#$pages) { + my $page = $pages->[$page_index]; + exact_keys($page, [qw(kind version encoding values rows)], + "$label page $page_index"); + require_string($page->{kind}, "$label page kind"); + require_string($page->{encoding}, "$label page encoding"); + require_nonnegative_integer($page->{values}, "$label page values"); + require_string($page->{version}, "$label page version") + if defined $page->{version}; + require_nonnegative_integer($page->{rows}, "$label page rows") + if defined $page->{rows}; + } + } + } + my $arrow = require_object($value->{arrow}, "$label Arrow"); + exact_keys($arrow, [qw(status schema canonical_rows json_rows ordered_map_rows + diagnostic)], "$label Arrow"); + require_string($arrow->{status}, "$label Arrow status"); + require_array($arrow->{canonical_rows}, "$label Arrow canonical rows"); + require_array($arrow->{json_rows}, "$label Arrow JSON rows"); +} + +sub parse_java { + my ($path, $unsupported) = @_; + my @input = lines($path); + fail("Java evidence is empty") unless @input; + my $run = parse_json_value(shift(@input), "Java run record"); + exact_keys($run, [qw(record schema_version oracle command parquet_java_version + parquet_java_commit avro_add_list_element_records fixture_writer java_version + java_vendor file_count supported_count unsupported_count)], "Java run record"); + my $writer = require_object($run->{fixture_writer}, "Java fixture writer"); + exact_keys($writer, [qw(compression dictionary page_size row_group_size + page_checksums validation page_versions)], "Java fixture writer"); + require_false($run->{avro_add_list_element_records}, + "Java Avro list-element setting"); + require_false($writer->{dictionary}, "Java fixture dictionary setting"); + fail("Java fixture page-checksum setting differs") unless + JSON::PP::is_bool($writer->{page_checksums}) && $writer->{page_checksums}; + fail("Java fixture validation setting differs") unless + JSON::PP::is_bool($writer->{validation}) && $writer->{validation}; + require_nonnegative_integer($run->{file_count}, "Java run file count"); + require_nonnegative_integer($run->{supported_count}, "Java supported count"); + require_nonnegative_integer($run->{unsupported_count}, "Java unsupported count"); + fail("Java run record differs") unless ($run->{record} // '') eq 'run' + && ($run->{schema_version} // 0) == 1 + && ($run->{oracle} // '') eq 'parquet-java' + && ($run->{command} // '') eq 'audit' + && ($run->{parquet_java_version} // '') eq '1.17.1' + && ($run->{parquet_java_commit} // '') eq + '78a8d3230eb4769db93de5f2f2e18363c04cae81' + && ($run->{java_version} // '') eq '11.0.28' + && ($run->{java_vendor} // '') eq 'Eclipse Adoptium' + && ($writer->{compression} // '') eq 'UNCOMPRESSED' + && ($writer->{page_size} // 0) == 1_048_576 + && ($writer->{row_group_size} // 0) == 134_217_728; + same_json($writer->{page_versions}, ['v1', 'v2'], + "Java fixture page versions"); + my %files; + my @order; + my $supported = 0; + my $unsupported_count = 0; + for my $index (0 .. $#input) { + my $record = parse_json_value($input[$index], "Java record @{[$index + 2]}"); + fail("Java record is not an object") unless ref($record) eq 'HASH'; + my $kind = $record->{record} // ''; + fail("unknown Java record kind $kind") + unless $kind eq 'file' || $kind eq 'unsupported'; + if ($kind eq 'file') { + validate_java_file_record($record, "Java record @{[$index + 2]}"); + } else { + exact_keys($record, [qw(record file error_class error_message)], + "Java record @{[$index + 2]}"); + } + my $file = normalized_path($record->{file} // '', "Java evidence file"); + fail("duplicate Java evidence file $file") if exists $files{$file}; + push @order, $file; + if ($kind eq 'unsupported') { + my $key = "parquet-java\0$file"; + fail("unexpected Java unsupported file $file") unless exists $unsupported->{$key}; + my ($class, $message) = @{$unsupported->{$key}}; + my $actual_message = defined $record->{error_message} + ? $record->{error_message} : ''; + fail("Java unsupported class differs for $file") + unless ($record->{error_class} // '') eq $class; + fail("Java unsupported message differs for $file") + unless $actual_message eq $message; + $unsupported_count++; + } else { + fail("allowlisted Java file became supported: $file") + if exists $unsupported->{"parquet-java\0$file"}; + $supported++; + } + $files{$file} = $record; + } + my @sorted = sort @order; + fail("Java evidence order is not deterministic") + unless join("\0", @order) eq join("\0", @sorted); + fail("Java run file count differs") unless ($run->{file_count} // -1) == @input; + fail("Java run supported count differs") + unless ($run->{supported_count} // -1) == $supported; + fail("Java run unsupported count differs") + unless ($run->{unsupported_count} // -1) == $unsupported_count; + return (\%files, $supported, $unsupported_count); +} + +sub parse_rust { + my ($path, $unsupported) = @_; + my $text = join("\n", lines($path)) . "\n"; + my $report = parse_json_value($text, "Rust evidence"); + exact_keys($report, [qw(evidence_version oracle action file_count supported_count + unsupported_count files)], "Rust evidence"); + my $oracle = $report->{oracle}; + exact_keys($oracle, [qw(name version commit rust_toolchain rustc cargo)], + "Rust oracle record"); + require_nonnegative_integer($report->{file_count}, "Rust run file count"); + require_nonnegative_integer($report->{supported_count}, "Rust supported count"); + require_nonnegative_integer($report->{unsupported_count}, "Rust unsupported count"); + fail("Rust oracle record differs") unless + ($report->{evidence_version} // 0) == 1 + && ($report->{action} // '') eq 'audit' + && ($oracle->{name} // '') eq 'arrow-rs' + && ($oracle->{version} // '') eq '59.2.0' + && ($oracle->{commit} // '') eq + '782e5a685501a9db6cc8e9a3b7cbff894940c47a' + && ($oracle->{rust_toolchain} // '') eq '1.96.1'; + my $records = $report->{files}; + fail("Rust files are not an array") unless ref($records) eq 'ARRAY'; + my %files; + my @order; + my $supported = 0; + my $unsupported_count = 0; + for my $record (@$records) { + exact_keys($record, [qw(status file evidence error)], "Rust file record"); + my $file = normalized_path($record->{file} // '', "Rust evidence file"); + fail("duplicate Rust evidence file $file") if exists $files{$file}; + push @order, $file; + my $status = $record->{status} // ''; + fail("unknown Rust status for $file") + unless $status eq 'supported' || $status eq 'unsupported'; + if ($status eq 'unsupported') { + my $key = "arrow-rs\0$file"; + fail("unexpected Rust unsupported file $file") unless exists $unsupported->{$key}; + my ($class, $message) = @{$unsupported->{$key}}; + fail("Rust unsupported class differs for $file") unless $class eq 'error'; + fail("Rust unsupported message differs for $file") + unless ($record->{error} // '') eq $message; + fail("Rust unsupported evidence is not empty") + if defined $record->{evidence}; + $unsupported_count++; + } else { + fail("allowlisted Rust file became supported: $file") + if exists $unsupported->{"arrow-rs\0$file"}; + fail("Rust supported evidence is absent") + unless ref($record->{evidence}) eq 'HASH'; + fail("Rust supported error is not null") if defined $record->{error}; + validate_rust_file_evidence($record->{evidence}, + "Rust evidence for $file"); + fail("Rust evidence case ID differs for $file") + unless $record->{evidence}->{case_id} eq $file; + my @parts = split m{/}, $file; + fail("Rust evidence file name differs for $file") + unless $record->{evidence}->{file_name} eq $parts[-1]; + $supported++; + } + $files{$file} = $record; + } + my @sorted = sort @order; + fail("Rust evidence order is not deterministic") + unless join("\0", @order) eq join("\0", @sorted); + fail("Rust run file count differs") + unless ($report->{file_count} // -1) == @$records; + fail("Rust run supported count differs") + unless ($report->{supported_count} // -1) == $supported; + fail("Rust run unsupported count differs") + unless ($report->{unsupported_count} // -1) == $unsupported_count; + return (\%files, $supported, $unsupported_count); +} + +my $canonical = JSON::PP->new->canonical->allow_nonref; + +sub expected_codec { + my ($codec) = @_; + my %values = ( + uncompressed => 'UNCOMPRESSED', + snappy => 'SNAPPY', + gzip => 'GZIP', + brotli => 'BROTLI', + zstd => 'ZSTD', + lz4_raw => 'LZ4_RAW', + ); + fail("unsupported expected codec $codec") unless exists $values{$codec}; + return $values{$codec}; +} + +sub same_json { + my ($left, $right, $label) = @_; + fail("$label differs") unless $canonical->encode($left) eq + $canonical->encode($right); +} + +sub java_file { + my ($record, $expected_hash, $label) = @_; + fail("$label is not supported") unless ($record->{record} // '') eq 'file'; + fail("$label SHA-256 differs") + unless ($record->{sha256} // '') eq $expected_hash; + return $record; +} + +sub compare_java { + my ($source, $target, $mapping) = @_; + for my $side (['reference', $source], ['target', $target]) { + my ($name, $record) = @$side; + fail("Java $name case ID differs for $mapping->{target}") + if defined($record->{case_id}) && + $record->{case_id} ne $mapping->{case_id}; + fail("Java $name page version differs for $mapping->{target}") + if defined($record->{page_version}) && + $record->{page_version} ne $mapping->{page_version}; + } + same_json($source->{physical_schema}, $target->{physical_schema}, + "Java physical schema for $mapping->{target}"); + same_json($source->{row_count}, $target->{row_count}, + "Java row count for $mapping->{target}"); + same_json($source->{raw_group_rows}, $target->{raw_group_rows}, + "Java raw rows for $mapping->{target}"); + my $source_columns = $source->{columns}; + my $target_columns = $target->{columns}; + fail("Java columns are not arrays") unless ref($source_columns) eq 'ARRAY' + && ref($target_columns) eq 'ARRAY'; + fail("Java column count differs for $mapping->{target}") + unless @$source_columns == @$target_columns; + for my $index (0 .. $#$source_columns) { + my $left = $source_columns->[$index]; + my $right = $target_columns->[$index]; + for my $field (qw(path physical_type logical_type type_length + max_repetition_level max_definition_level repetition definition dense)) { + same_json($left->{$field}, $right->{$field}, + "Java column $index $field for $mapping->{target}"); + } + fail("Java target dictionary is present for $mapping->{target}") + if @{$right->{dictionaries} // []}; + fail("Java reference dictionary is present for $mapping->{target}") + if @{$left->{dictionaries} // []}; + my $source_pages = $left->{pages}; + fail("Java reference pages are empty for $mapping->{target}") + unless ref($source_pages) eq 'ARRAY' && @$source_pages; + my $pages = $right->{pages}; + fail("Java target pages are empty for $mapping->{target}") + unless ref($pages) eq 'ARRAY' && @$pages; + my $expected_type = $mapping->{page_version} eq 'v1' + ? 'DATA_PAGE_V1' : 'DATA_PAGE_V2'; + for my $page (@$source_pages) { + fail("Java reference page type differs for $mapping->{target}") + unless ($page->{type} // '') eq $expected_type; + } + my ($values, $rows) = (0, 0); + for my $page (@$pages) { + fail("Java target page type differs for $mapping->{target}") + unless ($page->{type} // '') eq $expected_type; + fail("Java target page encoding differs for $mapping->{target}") + unless ($page->{encoding} // '') eq 'PLAIN'; + $values += $page->{value_count}; + $rows += $page->{row_count}; + } + fail("Java target page values differ for $mapping->{target}") + unless $values == @{$right->{repetition}}; + my $derived_rows = grep { $_ == 0 } @{$right->{repetition}}; + fail("Java target page rows differ for $mapping->{target}") + unless $rows == $derived_rows; + } + my $row_groups = require_array($target->{row_groups}, + "Java target row groups for $mapping->{target}"); + fail("Java target row groups are empty for $mapping->{target}") + unless @$row_groups; + for my $group (@$row_groups) { + my $chunks = require_array($group->{columns}, + "Java target row-group columns for $mapping->{target}"); + fail("Java target row-group columns are empty for $mapping->{target}") + unless @$chunks; + for my $column (@$chunks) { + fail("Java target codec differs for $mapping->{target}") + unless ($column->{codec} // '') eq expected_codec($mapping->{codec}); + } + } + my $source_groups = require_array($source->{row_groups}, + "Java reference row groups for $mapping->{target}"); + fail("Java reference row groups are empty for $mapping->{target}") + unless @$source_groups; + for my $group (@$source_groups) { + my $chunks = require_array($group->{columns}, + "Java reference row-group columns for $mapping->{target}"); + fail("Java reference row-group columns are empty for $mapping->{target}") + unless @$chunks; + for my $column (@$chunks) { + fail("Java reference codec differs for $mapping->{target}") + unless ($column->{codec} // '') eq 'UNCOMPRESSED'; + } + } + my $source_avro = $source->{avro}->{inferred}; + my $target_avro = $target->{avro}->{inferred}; + if (($source_avro->{status} // '') eq 'success') { + fail("Java Avro target rejected $mapping->{target}") + unless ($target_avro->{status} // '') eq 'success'; + same_json($source_avro->{materialized_schema}, + $target_avro->{materialized_schema}, + "Java Avro schema for $mapping->{target}"); + same_json($source_avro->{normalized_rows}, + $target_avro->{normalized_rows}, + "Java Avro rows for $mapping->{target}"); + } +} + +sub rust_file { + my ($record, $expected_hash, $label) = @_; + fail("$label is not supported") unless ($record->{status} // '') eq 'supported'; + my $evidence = $record->{evidence}; + fail("$label SHA-256 differs") + unless ($evidence->{sha256} // '') eq $expected_hash; + return $evidence; +} + +sub flattened { + my ($column, $field) = @_; + my @values; + for my $group (@{$column->{row_groups} // []}) { + push @values, @{$group->{$field} // []}; + } + return \@values; +} + +sub compare_rust { + my ($source, $target, $mapping) = @_; + same_json($source->{physical_schema}, $target->{physical_schema}, + "Rust physical schema for $mapping->{target}"); + same_json($source->{rows}, $target->{rows}, + "Rust row count for $mapping->{target}"); + my $source_columns = $source->{columns}; + my $target_columns = $target->{columns}; + fail("Rust columns are not arrays") unless ref($source_columns) eq 'ARRAY' + && ref($target_columns) eq 'ARRAY'; + fail("Rust column count differs for $mapping->{target}") + unless @$source_columns == @$target_columns; + for my $index (0 .. $#$source_columns) { + my $left = $source_columns->[$index]; + my $right = $target_columns->[$index]; + for my $field (qw(path physical_type maximum_definition_level + maximum_repetition_level)) { + same_json($left->{$field}, $right->{$field}, + "Rust column $index $field for $mapping->{target}"); + } + for my $field (qw(repetition definition dense_values)) { + same_json(flattened($left, $field), flattened($right, $field), + "Rust column $index $field for $mapping->{target}"); + } + my $source_groups = require_array($left->{row_groups}, + "Rust reference row groups for $mapping->{target}"); + fail("Rust reference row groups are empty for $mapping->{target}") + unless @$source_groups; + for my $group (@$source_groups) { + fail("Rust reference codec differs for $mapping->{target}") + unless ($group->{compression} // '') eq 'UNCOMPRESSED'; + my $source_pages = require_array($group->{pages}, + "Rust reference pages for $mapping->{target}"); + fail("Rust reference pages are empty for $mapping->{target}") + unless @$source_pages; + for my $page (@$source_pages) { + fail("Rust reference dictionary is present for $mapping->{target}") + unless ($page->{kind} // '') eq 'data'; + fail("Rust reference page version differs for $mapping->{target}") + unless ($page->{version} // '') eq $mapping->{page_version}; + } + } + my $groups = require_array($right->{row_groups}, + "Rust target row groups for $mapping->{target}"); + fail("Rust target row groups are empty for $mapping->{target}") + unless @$groups; + for my $group (@$groups) { + fail("Rust target codec differs for $mapping->{target}") + unless ($group->{compression} // '') eq + expected_codec($mapping->{codec}); + } + my @pages = map { @{$_->{pages} // []} } @{$right->{row_groups} // []}; + fail("Rust target pages are empty for $mapping->{target}") unless @pages; + my ($values, $rows) = (0, 0); + for my $page (@pages) { + fail("Rust target dictionary is present for $mapping->{target}") + unless ($page->{kind} // '') eq 'data'; + fail("Rust target page version differs for $mapping->{target}") + unless ($page->{version} // '') eq $mapping->{page_version}; + fail("Rust target page encoding differs for $mapping->{target}") + unless ($page->{encoding} // '') eq 'PLAIN'; + $values += $page->{values}; + $rows += $page->{rows}; + } + my $repetition = flattened($right, 'repetition'); + fail("Rust target page values differ for $mapping->{target}") + unless $values == @$repetition; + my $derived_rows = grep { $_ == 0 } @$repetition; + fail("Rust target page rows differ for $mapping->{target}") + unless $rows == $derived_rows; + } + my $source_arrow = $source->{arrow}; + my $target_arrow = $target->{arrow}; + if (($source_arrow->{status} // '') eq 'success') { + fail("Rust Arrow target rejected $mapping->{target}") + unless ($target_arrow->{status} // '') eq 'success'; + for my $field (qw(schema canonical_rows ordered_map_rows)) { + same_json($source_arrow->{$field}, $target_arrow->{$field}, + "Rust Arrow $field for $mapping->{target}"); + } + } +} + +my %args = arguments(); +my @mappings = parse_manifest($args{'--manifest'}); +my %unsupported = parse_unsupported($args{'--unsupported'}); +my ($java, $java_supported, $java_unsupported) = + parse_java($args{'--java'}, \%unsupported); +my ($rust, $rust_supported, $rust_unsupported) = + parse_rust($args{'--rust'}, \%unsupported); + +my %expected_files; +for my $mapping (@mappings) { + $expected_files{$mapping->{reference}} = 1; + $expected_files{$mapping->{target}} = 1; +} +fail("expected 352 unique Parquet inputs") unless keys(%expected_files) == 352; +for my $oracle (['Java', $java], ['Rust', $rust]) { + my ($label, $files) = @$oracle; + my @actual = sort keys %$files; + my @expected = sort keys %expected_files; + fail("$label input file set differs") + unless join("\0", @actual) eq join("\0", @expected); +} +for my $key (keys %unsupported) { + my ($oracle, $file) = split /\0/, $key, 2; + my $files = $oracle eq 'parquet-java' ? $java : $rust; + fail("unused unsupported allowlist entry $oracle/$file") + unless exists $files->{$file}; +} + +my ($java_pairs, $rust_pairs, $externally_supported, $paired_mappings) = + (0, 0, 0, 0); +my %kind_counts; +my %codec_counts; +for my $mapping (@mappings) { + $kind_counts{$mapping->{kind}}++; + $codec_counts{$mapping->{codec}}++ if $mapping->{kind} eq 'property'; + my $java_source = $java->{$mapping->{reference}}; + my $java_target = $java->{$mapping->{target}}; + my $rust_source = $rust->{$mapping->{reference}}; + my $rust_target = $rust->{$mapping->{target}}; + my $java_ok = ($java_target->{record} // '') eq 'file'; + my $rust_ok = ($rust_target->{status} // '') eq 'supported'; + my $paired = 0; + $externally_supported++ if $java_ok || $rust_ok; + if ($java_ok && ($java_source->{record} // '') eq 'file') { + compare_java( + java_file($java_source, $mapping->{reference_sha256}, 'Java reference'), + java_file($java_target, $mapping->{target_sha256}, 'Java target'), + $mapping); + $java_pairs++; + $paired = 1; + } + if ($rust_ok && ($rust_source->{status} // '') eq 'supported') { + compare_rust( + rust_file($rust_source, $mapping->{reference_sha256}, 'Rust reference'), + rust_file($rust_target, $mapping->{target_sha256}, 'Rust target'), + $mapping); + $rust_pairs++; + $paired = 1; + } + if ($mapping->{kind} ne 'provenance') { + fail("mapping has no successful paired external comparison: " + . "$mapping->{reference} -> $mapping->{target}") unless $paired; + $paired_mappings++; + } +} +my %expected_kind_counts = ( + binding => 24, + provenance => 2, + property => 192, + external => 38, +); +fail("fixture kind set differs") unless scalar(keys %kind_counts) == + scalar(keys %expected_kind_counts); +for my $kind (sort keys %expected_kind_counts) { + fail("fixture kind count differs for $kind") unless + ($kind_counts{$kind} // 0) == $expected_kind_counts{$kind}; +} +fail("property codec matrix differs") unless scalar(keys %codec_counts) == 6 + && !grep { $codec_counts{$_} != 32 } + qw(uncompressed snappy gzip brotli zstd lz4_raw); + +my $summary = { + schema_version => 1, + status => 'ok', + mappings => scalar(@mappings), + input_files => scalar(keys %expected_files), + kinds => \%kind_counts, + property_codecs => \%codec_counts, + java => { + supported_files => $java_supported, + unsupported_files => $java_unsupported, + compared_mappings => $java_pairs, + }, + rust => { + supported_files => $rust_supported, + unsupported_files => $rust_unsupported, + compared_mappings => $rust_pairs, + }, + mappings_with_external_success => $externally_supported, + paired_mappings => $paired_mappings, +}; +open my $output, '>:raw', $args{'--output'} + or fail("cannot open $args{'--output'}: $!"); +print {$output} JSON::PP->new->canonical->pretty->encode($summary) + or fail("cannot write $args{'--output'}: $!"); +close $output or fail("cannot close $args{'--output'}: $!"); +print "validated 256 N5 Julia mappings with Java and Rust\n"; diff --git a/test/conformance/n5/corpus-files.sha256 b/test/conformance/n5/corpus-files.sha256 new file mode 100644 index 0000000..f5559f6 --- /dev/null +++ b/test/conformance/n5/corpus-files.sha256 @@ -0,0 +1,14 @@ +44f29191b5fa8cfe0ab848495bd8ef89344ac0d8f87b3dff12e267631e2b5c03 data/datapage_v2.snappy.parquet +5591dde252b46bc238a88e9c02e35780c5eb086e2677df105aaa91ff1fde8fba data/incorrect_map_schema.parquet +1ce6839f093ebc0699b1e2769ed04036bab40405bacbb5dacdd376dd94c13451 data/large_string_map.brotli.parquet +5988ab91b6cb7efa7bf6a77f789b40929212280519be6c9daad56e01d5ceb218 data/list_columns.parquet +5c4fc6c13fe7308acb2fd317a3bd59e5b9c9c206c005e863ae0a1abdbbf5e2ea data/map_no_value.parquet +2cb2cc0564486a28550429a8b6d0907bbb41e138546797bc91a4ebd850edd5a5 data/nested_lists.snappy.parquet +db1a493003a7dcd2011bf89e460fed007903fcdeb58f53df29387b4e908e2a6d data/nested_maps.snappy.parquet +48427178bfef9e6edd9018f2ef7b084077c00057234a780271a8220ca53b33da data/nested_structs.rust.parquet +e7927cde24c083e42a3d4b37ac962d34381f71c2d252169b627dd8459a5880e3 data/nonnullable.impala.parquet +e64a64ff130c8dff64a6bc41480c51c87918d5e63bc75167b58524aa0fa01496 data/null_list.parquet +de9102a599d852be3af1d2af5d3498d8e019c329096a6f2d260f55ae2d6ed0ae data/nullable.impala.parquet +065b336c65885ab9dfd97cf85ce39a45488ed12d0183917db0a11621b0711e3b data/old_list_structure.parquet +97d35acb9721e40fc0f66fba916a442c4a1cf77a35992dc891ad0cdcc5a24cfb data/repeated_no_annotation.parquet +fcd6152058b8b8259a516105da5919b23cb8ccfc42258de0fe20e3107f8ef809 data/repeated_primitive_no_list.parquet diff --git a/test/conformance/n5/corpus.jl b/test/conformance/n5/corpus.jl new file mode 100644 index 0000000..ecac685 --- /dev/null +++ b/test/conformance/n5/corpus.jl @@ -0,0 +1,67 @@ +using SHA +using Test + +const N5_CORPUS_COMMIT = + "09f3cdbde45302f0f0c689c950e465e98a9df960" + +const N5_CORPUS_FILES = [ + ("list_columns.parquet", + "5988ab91b6cb7efa7bf6a77f789b40929212280519be6c9daad56e01d5ceb218"), + ("null_list.parquet", + "e64a64ff130c8dff64a6bc41480c51c87918d5e63bc75167b58524aa0fa01496"), + ("datapage_v2.snappy.parquet", + "44f29191b5fa8cfe0ab848495bd8ef89344ac0d8f87b3dff12e267631e2b5c03"), + ("old_list_structure.parquet", + "065b336c65885ab9dfd97cf85ce39a45488ed12d0183917db0a11621b0711e3b"), + ("nested_lists.snappy.parquet", + "2cb2cc0564486a28550429a8b6d0907bbb41e138546797bc91a4ebd850edd5a5"), + ("nested_maps.snappy.parquet", + "db1a493003a7dcd2011bf89e460fed007903fcdeb58f53df29387b4e908e2a6d"), + ("repeated_primitive_no_list.parquet", + "fcd6152058b8b8259a516105da5919b23cb8ccfc42258de0fe20e3107f8ef809"), + ("repeated_no_annotation.parquet", + "97d35acb9721e40fc0f66fba916a442c4a1cf77a35992dc891ad0cdcc5a24cfb"), + ("nullable.impala.parquet", + "de9102a599d852be3af1d2af5d3498d8e019c329096a6f2d260f55ae2d6ed0ae"), + ("nonnullable.impala.parquet", + "e7927cde24c083e42a3d4b37ac962d34381f71c2d252169b627dd8459a5880e3"), + ("map_no_value.parquet", + "5c4fc6c13fe7308acb2fd317a3bd59e5b9c9c206c005e863ae0a1abdbbf5e2ea"), + ("incorrect_map_schema.parquet", + "5591dde252b46bc238a88e9c02e35780c5eb086e2677df105aaa91ff1fde8fba"), + ("nested_structs.rust.parquet", + "48427178bfef9e6edd9018f2ef7b084077c00057234a780271a8220ca53b33da"), + ("large_string_map.brotli.parquet", + "1ce6839f093ebc0699b1e2769ed04036bab40405bacbb5dacdd376dd94c13451"), +] + +function n5corpusroot() + default = normpath(joinpath(@__DIR__, "..", "..", "parquet-testing")) + return get(ENV, "PARQUET_TESTING_DIR", default) +end + +function n5corpusrevision(root::String) + try + return readchomp(`git -C $root rev-parse HEAD`) + catch err + throw(ErrorException("cannot verify parquet-testing revision at " * + "$(repr(root)): $(sprint(showerror, err))")) + end +end + +function n5filesha256(path::String) + return open(path, "r") do input + return bytes2hex(SHA.sha256(input)) + end +end + +@testset "N5 pinned nested corpus" begin + root = n5corpusroot() + @test isdir(root) + @test n5corpusrevision(root) == N5_CORPUS_COMMIT + for (name, expected) in N5_CORPUS_FILES + path = joinpath(root, "data", name) + @test isfile(path) + @test n5filesha256(path) == expected + end +end diff --git a/test/conformance/n5/expected/arrow-rs-rule3-near-neighbor.json b/test/conformance/n5/expected/arrow-rs-rule3-near-neighbor.json new file mode 100644 index 0000000..504de1e --- /dev/null +++ b/test/conformance/n5/expected/arrow-rs-rule3-near-neighbor.json @@ -0,0 +1,274 @@ +{ + "evidence_version": 1, + "oracle": { + "name": "arrow-rs", + "version": "59.2.0", + "commit": "782e5a685501a9db6cc8e9a3b7cbff894940c47a", + "rust_toolchain": "1.96.1", + "rustc": "rustc 1.96.1 (31fca3adb 2026-06-26)", + "cargo": "cargo 1.96.1 (356927216 2026-06-26)" + }, + "action": "diagnose-rule3-near-neighbor", + "files": [ + { + "case_id": "arrow-rs-list-rule3-unannotated-near-neighbor", + "file_name": "arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet", + "sha256": "40eb4521da12a9bd5db38cfe9e1f31dd91bd1095ec5b24b94f9a02318e40128f", + "file_bytes": 243, + "rows": 4, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group values (LIST) {\n REPEATED group list {\n REPEATED INT32 element;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "values", + "list", + "element" + ], + "physical_type": "INT32", + "maximum_definition_level": 3, + "maximum_repetition_level": 2, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 2, + 1, + 1 + ], + "definition": [ + 0, + 1, + 2, + 3, + 3, + 2, + 3 + ], + "dense_values": [ + 1, + 2, + 3 + ], + "pages": [ + { + "kind": "data", + "version": "v1", + "encoding": "PLAIN", + "values": 7, + "rows": 4 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "values", + "nullable": true, + "data_type": { + "list": { + "data_type": { + "list": { + "data_type": "Int32", + "metadata": {}, + "name": "element", + "nullable": false + } + }, + "metadata": {}, + "name": "element", + "nullable": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "values", + "value": null + } + ], + [ + { + "field": "values", + "value": [] + } + ], + [ + { + "field": "values", + "value": [ + [] + ] + } + ], + [ + { + "field": "values", + "value": [ + [ + 1, + 2 + ], + [], + [ + 3 + ] + ] + } + ] + ], + "json_rows": [ + "{\"values\":null}", + "{\"values\":[]}", + "{\"values\":[[]]}", + "{\"values\":[[1,2],[],[3]]}" + ], + "ordered_map_rows": null, + "diagnostic": null + } + }, + { + "case_id": "arrow-rs-list-rule3-unannotated-near-neighbor", + "file_name": "arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet", + "sha256": "4fe0319ea46248612489d845eb6fdbfa127077f06a9e7a2efc14fa67def66ce1", + "file_bytes": 238, + "rows": 4, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group values (LIST) {\n REPEATED group list {\n REPEATED INT32 element;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "values", + "list", + "element" + ], + "physical_type": "INT32", + "maximum_definition_level": 3, + "maximum_repetition_level": 2, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 2, + 1, + 1 + ], + "definition": [ + 0, + 1, + 2, + 3, + 3, + 2, + 3 + ], + "dense_values": [ + 1, + 2, + 3 + ], + "pages": [ + { + "kind": "data", + "version": "v2", + "encoding": "DELTA_BINARY_PACKED", + "values": 7, + "rows": 4 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "values", + "nullable": true, + "data_type": { + "list": { + "data_type": { + "list": { + "data_type": "Int32", + "metadata": {}, + "name": "element", + "nullable": false + } + }, + "metadata": {}, + "name": "element", + "nullable": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "values", + "value": null + } + ], + [ + { + "field": "values", + "value": [] + } + ], + [ + { + "field": "values", + "value": [ + [] + ] + } + ], + [ + { + "field": "values", + "value": [ + [ + 1, + 2 + ], + [], + [ + 3 + ] + ] + } + ] + ], + "json_rows": [ + "{\"values\":null}", + "{\"values\":[]}", + "{\"values\":[[]]}", + "{\"values\":[[1,2],[],[3]]}" + ], + "ordered_map_rows": null, + "diagnostic": null + } + } + ] +} diff --git a/test/conformance/n5/expected/arrow-rs.json b/test/conformance/n5/expected/arrow-rs.json new file mode 100644 index 0000000..11711e1 --- /dev/null +++ b/test/conformance/n5/expected/arrow-rs.json @@ -0,0 +1,1164 @@ +{ + "evidence_version": 1, + "oracle": { + "name": "arrow-rs", + "version": "59.2.0", + "commit": "782e5a685501a9db6cc8e9a3b7cbff894940c47a", + "rust_toolchain": "1.96.1", + "rustc": "rustc 1.96.1 (31fca3adb 2026-06-26)", + "cargo": "cargo 1.96.1 (356927216 2026-06-26)" + }, + "action": "generate", + "files": [ + { + "case_id": "arrow-rs-duplicate-keys", + "file_name": "arrow-rs-duplicate-keys_v1.parquet", + "sha256": "acf5536a16167a2a870dfd55ad2a5d208712069d8e0f034cf396fa6d6c193184", + "file_bytes": 400, + "rows": 5, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group entries (MAP) {\n REPEATED group key_value {\n REQUIRED BYTE_ARRAY key (STRING);\n OPTIONAL INT32 value;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "entries", + "key_value", + "key" + ], + "physical_type": "BYTE_ARRAY", + "maximum_definition_level": 2, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 5, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1, + 1, + 0 + ], + "definition": [ + 0, + 1, + 2, + 2, + 2, + 2, + 2 + ], + "dense_values": [ + "a", + "a", + "a", + "b", + "c" + ], + "pages": [ + { + "kind": "data", + "version": "v1", + "encoding": "PLAIN", + "values": 7, + "rows": 5 + } + ] + } + ] + }, + { + "path": [ + "entries", + "key_value", + "value" + ], + "physical_type": "INT32", + "maximum_definition_level": 3, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 5, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1, + 1, + 0 + ], + "definition": [ + 0, + 1, + 2, + 3, + 3, + 3, + 3 + ], + "dense_values": [ + 1, + 2, + 3, + 4 + ], + "pages": [ + { + "kind": "data", + "version": "v1", + "encoding": "PLAIN", + "values": 7, + "rows": 5 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "entries", + "nullable": true, + "data_type": { + "map": { + "entries": { + "data_type": { + "struct": [ + { + "data_type": "Utf8", + "metadata": {}, + "name": "key", + "nullable": false + }, + { + "data_type": "Int32", + "metadata": {}, + "name": "value", + "nullable": true + } + ] + }, + "metadata": {}, + "name": "key_value", + "nullable": false + }, + "sorted": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "entries", + "value": null + } + ], + [ + { + "field": "entries", + "value": [] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "a", + "value": null + } + ] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "a", + "value": 1 + }, + { + "key": "a", + "value": 2 + }, + { + "key": "b", + "value": 3 + } + ] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "c", + "value": 4 + } + ] + } + ] + ], + "json_rows": [ + "{\"entries\":null}", + "{\"entries\":{}}", + "{\"entries\":{\"a\":null}}", + "{\"entries\":{\"a\":1,\"a\":2,\"b\":3}}", + "{\"entries\":{\"c\":4}}" + ], + "ordered_map_rows": [ + null, + [], + [ + { + "key": "a", + "value": null + } + ], + [ + { + "key": "a", + "value": 1 + }, + { + "key": "a", + "value": 2 + }, + { + "key": "b", + "value": 3 + } + ], + [ + { + "key": "c", + "value": 4 + } + ] + ], + "diagnostic": null + } + }, + { + "case_id": "arrow-rs-duplicate-keys", + "file_name": "arrow-rs-duplicate-keys_v2.parquet", + "sha256": "baaf9a09bc397db8c0b1d0453fa2d87e36f6a74bbc56778be39d27304007cbb3", + "file_bytes": 407, + "rows": 5, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group entries (MAP) {\n REPEATED group key_value {\n REQUIRED BYTE_ARRAY key (STRING);\n OPTIONAL INT32 value;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "entries", + "key_value", + "key" + ], + "physical_type": "BYTE_ARRAY", + "maximum_definition_level": 2, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 5, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1, + 1, + 0 + ], + "definition": [ + 0, + 1, + 2, + 2, + 2, + 2, + 2 + ], + "dense_values": [ + "a", + "a", + "a", + "b", + "c" + ], + "pages": [ + { + "kind": "data", + "version": "v2", + "encoding": "DELTA_BYTE_ARRAY", + "values": 7, + "rows": 5 + } + ] + } + ] + }, + { + "path": [ + "entries", + "key_value", + "value" + ], + "physical_type": "INT32", + "maximum_definition_level": 3, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 5, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1, + 1, + 0 + ], + "definition": [ + 0, + 1, + 2, + 3, + 3, + 3, + 3 + ], + "dense_values": [ + 1, + 2, + 3, + 4 + ], + "pages": [ + { + "kind": "data", + "version": "v2", + "encoding": "DELTA_BINARY_PACKED", + "values": 7, + "rows": 5 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "entries", + "nullable": true, + "data_type": { + "map": { + "entries": { + "data_type": { + "struct": [ + { + "data_type": "Utf8", + "metadata": {}, + "name": "key", + "nullable": false + }, + { + "data_type": "Int32", + "metadata": {}, + "name": "value", + "nullable": true + } + ] + }, + "metadata": {}, + "name": "key_value", + "nullable": false + }, + "sorted": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "entries", + "value": null + } + ], + [ + { + "field": "entries", + "value": [] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "a", + "value": null + } + ] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "a", + "value": 1 + }, + { + "key": "a", + "value": 2 + }, + { + "key": "b", + "value": 3 + } + ] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "c", + "value": 4 + } + ] + } + ] + ], + "json_rows": [ + "{\"entries\":null}", + "{\"entries\":{}}", + "{\"entries\":{\"a\":null}}", + "{\"entries\":{\"a\":1,\"a\":2,\"b\":3}}", + "{\"entries\":{\"c\":4}}" + ], + "ordered_map_rows": [ + null, + [], + [ + { + "key": "a", + "value": null + } + ], + [ + { + "key": "a", + "value": 1 + }, + { + "key": "a", + "value": 2 + }, + { + "key": "b", + "value": 3 + } + ], + [ + { + "key": "c", + "value": 4 + } + ] + ], + "diagnostic": null + } + }, + { + "case_id": "arrow-rs-optional-key-present", + "file_name": "arrow-rs-optional-key-present_v1.parquet", + "sha256": "a33be66fe1a5e7189c457b49163e5a4b658228ffb5054a40cd22f20f731ba21a", + "file_bytes": 386, + "rows": 4, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group entries (MAP) {\n REPEATED group key_value {\n OPTIONAL BYTE_ARRAY key (STRING);\n REQUIRED INT32 value;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "entries", + "key_value", + "key" + ], + "physical_type": "BYTE_ARRAY", + "maximum_definition_level": 3, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1 + ], + "definition": [ + 0, + 1, + 3, + 3, + 3 + ], + "dense_values": [ + "a", + "b", + "c" + ], + "pages": [ + { + "kind": "data", + "version": "v1", + "encoding": "PLAIN", + "values": 5, + "rows": 4 + } + ] + } + ] + }, + { + "path": [ + "entries", + "key_value", + "value" + ], + "physical_type": "INT32", + "maximum_definition_level": 2, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1 + ], + "definition": [ + 0, + 1, + 2, + 2, + 2 + ], + "dense_values": [ + 1, + 2, + 3 + ], + "pages": [ + { + "kind": "data", + "version": "v1", + "encoding": "PLAIN", + "values": 5, + "rows": 4 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "entries", + "nullable": true, + "data_type": { + "map": { + "entries": { + "data_type": { + "struct": [ + { + "data_type": "Utf8", + "metadata": {}, + "name": "key", + "nullable": false + }, + { + "data_type": "Int32", + "metadata": {}, + "name": "value", + "nullable": false + } + ] + }, + "metadata": {}, + "name": "key_value", + "nullable": false + }, + "sorted": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "entries", + "value": null + } + ], + [ + { + "field": "entries", + "value": [] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "a", + "value": 1 + } + ] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "b", + "value": 2 + }, + { + "key": "c", + "value": 3 + } + ] + } + ] + ], + "json_rows": [ + "{\"entries\":null}", + "{\"entries\":{}}", + "{\"entries\":{\"a\":1}}", + "{\"entries\":{\"b\":2,\"c\":3}}" + ], + "ordered_map_rows": [ + null, + [], + [ + { + "key": "a", + "value": 1 + } + ], + [ + { + "key": "b", + "value": 2 + }, + { + "key": "c", + "value": 3 + } + ] + ], + "diagnostic": null + } + }, + { + "case_id": "arrow-rs-optional-key-present", + "file_name": "arrow-rs-optional-key-present_v2.parquet", + "sha256": "f010b919bcc2d4e8c42f6d97df651266de06b6352f6b993756fd0a007e1c5028", + "file_bytes": 386, + "rows": 4, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group entries (MAP) {\n REPEATED group key_value {\n OPTIONAL BYTE_ARRAY key (STRING);\n REQUIRED INT32 value;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "entries", + "key_value", + "key" + ], + "physical_type": "BYTE_ARRAY", + "maximum_definition_level": 3, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1 + ], + "definition": [ + 0, + 1, + 3, + 3, + 3 + ], + "dense_values": [ + "a", + "b", + "c" + ], + "pages": [ + { + "kind": "data", + "version": "v2", + "encoding": "DELTA_BYTE_ARRAY", + "values": 5, + "rows": 4 + } + ] + } + ] + }, + { + "path": [ + "entries", + "key_value", + "value" + ], + "physical_type": "INT32", + "maximum_definition_level": 2, + "maximum_repetition_level": 1, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 1 + ], + "definition": [ + 0, + 1, + 2, + 2, + 2 + ], + "dense_values": [ + 1, + 2, + 3 + ], + "pages": [ + { + "kind": "data", + "version": "v2", + "encoding": "DELTA_BINARY_PACKED", + "values": 5, + "rows": 4 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "entries", + "nullable": true, + "data_type": { + "map": { + "entries": { + "data_type": { + "struct": [ + { + "data_type": "Utf8", + "metadata": {}, + "name": "key", + "nullable": false + }, + { + "data_type": "Int32", + "metadata": {}, + "name": "value", + "nullable": false + } + ] + }, + "metadata": {}, + "name": "key_value", + "nullable": false + }, + "sorted": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "entries", + "value": null + } + ], + [ + { + "field": "entries", + "value": [] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "a", + "value": 1 + } + ] + } + ], + [ + { + "field": "entries", + "value": [ + { + "key": "b", + "value": 2 + }, + { + "key": "c", + "value": 3 + } + ] + } + ] + ], + "json_rows": [ + "{\"entries\":null}", + "{\"entries\":{}}", + "{\"entries\":{\"a\":1}}", + "{\"entries\":{\"b\":2,\"c\":3}}" + ], + "ordered_map_rows": [ + null, + [], + [ + { + "key": "a", + "value": 1 + } + ], + [ + { + "key": "b", + "value": 2 + }, + { + "key": "c", + "value": 3 + } + ] + ], + "diagnostic": null + } + }, + { + "case_id": "arrow-rs-list-rule3", + "file_name": "arrow-rs-list-rule3_v1.parquet", + "sha256": "916475c20dde6afe338d60e43d2e4a6293c2bc26efb6dd439d1d7d9299834bc6", + "file_bytes": 247, + "rows": 4, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group values (LIST) {\n REPEATED group array (LIST) {\n REPEATED INT32 array;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "values", + "array", + "array" + ], + "physical_type": "INT32", + "maximum_definition_level": 3, + "maximum_repetition_level": 2, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 2, + 1, + 1 + ], + "definition": [ + 0, + 1, + 2, + 3, + 3, + 2, + 3 + ], + "dense_values": [ + 1, + 2, + 3 + ], + "pages": [ + { + "kind": "data", + "version": "v1", + "encoding": "PLAIN", + "values": 7, + "rows": 4 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "values", + "nullable": true, + "data_type": { + "list": { + "data_type": { + "list": { + "data_type": "Int32", + "metadata": {}, + "name": "array", + "nullable": false + } + }, + "metadata": {}, + "name": "array", + "nullable": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "values", + "value": null + } + ], + [ + { + "field": "values", + "value": [] + } + ], + [ + { + "field": "values", + "value": [ + [] + ] + } + ], + [ + { + "field": "values", + "value": [ + [ + 1, + 2 + ], + [], + [ + 3 + ] + ] + } + ] + ], + "json_rows": [ + "{\"values\":null}", + "{\"values\":[]}", + "{\"values\":[[]]}", + "{\"values\":[[1,2],[],[3]]}" + ], + "ordered_map_rows": null, + "diagnostic": null + } + }, + { + "case_id": "arrow-rs-list-rule3", + "file_name": "arrow-rs-list-rule3_v2.parquet", + "sha256": "ef72761c509f8bc302986d130f6d7d9382b975c8a3462a955c599bd5f427edac", + "file_bytes": 242, + "rows": 4, + "row_groups": 1, + "physical_schema": "message schema {\n OPTIONAL group values (LIST) {\n REPEATED group array (LIST) {\n REPEATED INT32 array;\n }\n }\n}\n", + "columns": [ + { + "path": [ + "values", + "array", + "array" + ], + "physical_type": "INT32", + "maximum_definition_level": 3, + "maximum_repetition_level": 2, + "row_groups": [ + { + "row_group": 0, + "rows": 4, + "compression": "UNCOMPRESSED", + "repetition": [ + 0, + 0, + 0, + 0, + 2, + 1, + 1 + ], + "definition": [ + 0, + 1, + 2, + 3, + 3, + 2, + 3 + ], + "dense_values": [ + 1, + 2, + 3 + ], + "pages": [ + { + "kind": "data", + "version": "v2", + "encoding": "DELTA_BINARY_PACKED", + "values": 7, + "rows": 4 + } + ] + } + ] + } + ], + "arrow": { + "status": "ok", + "schema": [ + { + "name": "values", + "nullable": true, + "data_type": { + "list": { + "data_type": { + "list": { + "data_type": "Int32", + "metadata": {}, + "name": "array", + "nullable": false + } + }, + "metadata": {}, + "name": "array", + "nullable": false + } + }, + "metadata": {} + } + ], + "canonical_rows": [ + [ + { + "field": "values", + "value": null + } + ], + [ + { + "field": "values", + "value": [] + } + ], + [ + { + "field": "values", + "value": [ + [] + ] + } + ], + [ + { + "field": "values", + "value": [ + [ + 1, + 2 + ], + [], + [ + 3 + ] + ] + } + ] + ], + "json_rows": [ + "{\"values\":null}", + "{\"values\":[]}", + "{\"values\":[[]]}", + "{\"values\":[[1,2],[],[3]]}" + ], + "ordered_map_rows": null, + "diagnostic": null + } + } + ] +} diff --git a/test/conformance/n5/expected/parquet-java.jsonl b/test/conformance/n5/expected/parquet-java.jsonl new file mode 100644 index 0000000..eed119b --- /dev/null +++ b/test/conformance/n5/expected/parquet-java.jsonl @@ -0,0 +1,31 @@ +{"record":"run","schema_version":1,"oracle":"parquet-java","command":"generate","parquet_java_version":"1.17.1","parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","avro_add_list_element_records":false,"fixture_writer":{"compression":"UNCOMPRESSED","dictionary":false,"page_size":1048576,"row_group_size":134217728,"page_checksums":true,"validation":true,"page_versions":["v1","v2"]},"java_version":"11.0.28","java_vendor":"Eclipse Adoptium","file_count":30} +{"record":"file","file":"list_rule1_primitive.v1.parquet","sha256":"3466fa00347fae5832a8b751cd50d74e0af0df490d7262bdccf36f0c2f38c2db","case_id":"list_rule1_primitive","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule1_primitive","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule1_primitive {\n optional group items (LIST) {\n repeated int32 element;\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":48,"columns":[{"path":"items.element","value_count":5,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":48,"total_uncompressed_size":48}]}],"raw_group_rows":["G{items=null}","G{items=G{element=[]}}","G{items=G{element=[i32:10]}}","G{items=G{element=[i32:20,i32:30]}}"],"columns":[{"path":"items.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":25,"uncompressed_size":25}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":25,"uncompressed_size":25}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule1_primitive\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule1_primitive.v2.parquet","sha256":"056989f7d112e735fc4f4e835278ba58610e522e1aa3a3e05afdaf540c0463cd","case_id":"list_rule1_primitive","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule1_primitive","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule1_primitive {\n optional group items (LIST) {\n repeated int32 element;\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":43,"columns":[{"path":"items.element","value_count":5,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{items=null}","G{items=G{element=[]}}","G{items=G{element=[i32:10]}}","G{items=G{element=[i32:20,i32:30]}}"],"columns":[{"path":"items.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":2,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":2,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule1_primitive\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule2_struct.v1.parquet","sha256":"b3c6ddd69b4b75bfe0ba1fcb21aca6db5d7b7c083fda0cd106b7229baa0a2f7b","case_id":"list_rule2_struct","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule2_struct","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule2_struct {\n optional group items (LIST) {\n repeated group element {\n required int32 x;\n optional int32 y;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":92,"columns":[{"path":"items.element.x","value_count":5,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":48,"total_uncompressed_size":48},{"path":"items.element.y","value_count":5,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":44,"total_uncompressed_size":44}]}],"raw_group_rows":["G{items=null}","G{items=G{element=[]}}","G{items=G{element=[G{x=i32:1,y=null}]}}","G{items=G{element=[G{x=i32:2,y=i32:20},G{x=i32:3,y=i32:30}]}}"],"columns":[{"path":"items.element.x","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":25,"uncompressed_size":25}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":25,"uncompressed_size":25}]}]},{"path":"items.element.y","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,3],"dense":["20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":21,"uncompressed_size":21}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,3],"dense":["20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":21,"uncompressed_size":21}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule2_struct\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"element\",\"fields\":[{\"name\":\"x\",\"type\":\"int\"},{\"name\":\"y\",\"type\":[\"null\",\"int\"],\"default\":null}]}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"x\":1,\"y\":null}]}","{\"items\":[{\"x\":2,\"y\":20},{\"x\":3,\"y\":30}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"x\":1,\"y\":null}]}","{\"items\":[{\"x\":2,\"y\":20},{\"x\":3,\"y\":30}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule2_struct.v2.parquet","sha256":"2b059b3b5b375c736e7d0abb69a5f1e8dc48606008ffd9e3910f34d3543fc6a6","case_id":"list_rule2_struct","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule2_struct","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule2_struct {\n optional group items (LIST) {\n repeated group element {\n required int32 x;\n optional int32 y;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":86,"columns":[{"path":"items.element.x","value_count":5,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43},{"path":"items.element.y","value_count":5,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{items=null}","G{items=G{element=[]}}","G{items=G{element=[G{x=i32:1,y=null}]}}","G{items=G{element=[G{x=i32:2,y=i32:20},G{x=i32:3,y=i32:30}]}}"],"columns":[{"path":"items.element.x","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":2,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":2,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]},{"path":"items.element.y","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,3],"dense":["20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,3],"dense":["20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule2_struct\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"element\",\"fields\":[{\"name\":\"x\",\"type\":\"int\"},{\"name\":\"y\",\"type\":[\"null\",\"int\"],\"default\":null}]}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"x\":1,\"y\":null}]}","{\"items\":[{\"x\":2,\"y\":20},{\"x\":3,\"y\":30}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"x\":1,\"y\":null}]}","{\"items\":[{\"x\":2,\"y\":20},{\"x\":3,\"y\":30}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule3_nested.v1.parquet","sha256":"c85821a7f6593ad3092f2efd3c8acc602f4969eb8dc36d49f1868b22f1d7d56a","case_id":"list_rule3_nested","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule3_nested","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule3_nested {\n optional group items (LIST) {\n repeated group array (LIST) {\n repeated int32 array;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":49,"columns":[{"path":"items.array.array","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":49,"total_uncompressed_size":49}]}],"raw_group_rows":["G{items=null}","G{items=G{array=[]}}","G{items=G{array=[G{array=[]}]}}","G{items=G{array=[G{array=[i32:1,i32:2]},G{array=[]},G{array=[i32:3]}]}}"],"columns":[{"path":"items.array.array","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule3_nested\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"array\",\"items\":\"int\"}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[[]]}","{\"items\":[[1,2],[],[3]]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[[]]}","{\"items\":[[1,2],[],[3]]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule3_nested.v2.parquet","sha256":"5333c37c239ae57781c63eec972537eb428e275da8d3cca0634acb963989e73e","case_id":"list_rule3_nested","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule3_nested","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule3_nested {\n optional group items (LIST) {\n repeated group array (LIST) {\n repeated int32 array;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":44,"columns":[{"path":"items.array.array","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":44,"total_uncompressed_size":44}]}],"raw_group_rows":["G{items=null}","G{items=G{array=[]}}","G{items=G{array=[G{array=[]}]}}","G{items=G{array=[G{array=[i32:1,i32:2]},G{array=[]},G{array=[i32:3]}]}}"],"columns":[{"path":"items.array.array","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule3_nested\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"array\",\"items\":\"int\"}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[[]]}","{\"items\":[[1,2],[],[3]]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[[]]}","{\"items\":[[1,2],[],[3]]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule3_unannotated_diagnostic.v1.parquet","sha256":"ab2c1a962ee992845540c6aa04056f15cdc4b40cf89627fb5160dd3fb07a5123","case_id":"list_rule3_unannotated_diagnostic","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule3_unannotated_diagnostic","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule3_unannotated_diagnostic {\n optional group items (LIST) {\n repeated group list {\n repeated int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":49,"columns":[{"path":"items.list.element","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":49,"total_uncompressed_size":49}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=[]}]}}","G{items=G{list=[G{element=[i32:1,i32:2]},G{element=[]},G{element=[i32:3]}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"rejected","read_schema":null,"materialized_schema":null,"rows":[],"normalized_rows":[],"error_class":"java.lang.ClassCastException","error_message":"repeated int32 element is not a group","error_stack":["org.apache.parquet.schema.Type.asGroupType(Type.java:247)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:426)","org.apache.parquet.avro.AvroRecordConverter.(AvroRecordConverter.java:151)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:415)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:337)","org.apache.parquet.avro.AvroRecordConverter$AvroCollectionConverter.(AvroRecordConverter.java:613)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:426)","org.apache.parquet.avro.AvroRecordConverter.(AvroRecordConverter.java:151)","org.apache.parquet.avro.AvroRecordConverter.(AvroRecordConverter.java:98)","org.apache.parquet.avro.AvroRecordMaterializer.(AvroRecordMaterializer.java:33)","org.apache.parquet.avro.AvroReadSupport.prepareForRead(AvroReadSupport.java:195)","org.apache.parquet.hadoop.InternalParquetRecordReader.initialize(InternalParquetRecordReader.java:205)","org.apache.parquet.hadoop.ParquetReader.initReader(ParquetReader.java:170)","org.apache.parquet.hadoop.ParquetReader.read(ParquetReader.java:139)","org.julialang.parquet.n5.ParquetEvidence.readAvro(ParquetEvidence.java:509)","org.julialang.parquet.n5.ParquetEvidence.inspect(ParquetEvidence.java:283)","org.julialang.parquet.n5.OracleMain.generateFixtures(OracleMain.java:258)","org.julialang.parquet.n5.OracleMain.runGenerate(OracleMain.java:121)","org.julialang.parquet.n5.OracleMain.run(OracleMain.java:98)","org.julialang.parquet.n5.OracleMain.main(OracleMain.java:77)"],"exception_chain":[{"class":"java.lang.ClassCastException","message":"repeated int32 element is not a group"}]},"explicit":null}} +{"record":"file","file":"list_rule3_unannotated_diagnostic.v2.parquet","sha256":"51ece6eccb1123e817c7e4a9f5f0ff235299501c7e7aa4e78f2878ecb4ae4e5f","case_id":"list_rule3_unannotated_diagnostic","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule3_unannotated_diagnostic","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule3_unannotated_diagnostic {\n optional group items (LIST) {\n repeated group list {\n repeated int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":44,"columns":[{"path":"items.list.element","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":44,"total_uncompressed_size":44}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=[]}]}}","G{items=G{list=[G{element=[i32:1,i32:2]},G{element=[]},G{element=[i32:3]}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","2","3"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"rejected","read_schema":null,"materialized_schema":null,"rows":[],"normalized_rows":[],"error_class":"java.lang.ClassCastException","error_message":"repeated int32 element is not a group","error_stack":["org.apache.parquet.schema.Type.asGroupType(Type.java:247)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:426)","org.apache.parquet.avro.AvroRecordConverter.(AvroRecordConverter.java:151)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:415)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:337)","org.apache.parquet.avro.AvroRecordConverter$AvroCollectionConverter.(AvroRecordConverter.java:613)","org.apache.parquet.avro.AvroRecordConverter.newConverter(AvroRecordConverter.java:426)","org.apache.parquet.avro.AvroRecordConverter.(AvroRecordConverter.java:151)","org.apache.parquet.avro.AvroRecordConverter.(AvroRecordConverter.java:98)","org.apache.parquet.avro.AvroRecordMaterializer.(AvroRecordMaterializer.java:33)","org.apache.parquet.avro.AvroReadSupport.prepareForRead(AvroReadSupport.java:195)","org.apache.parquet.hadoop.InternalParquetRecordReader.initialize(InternalParquetRecordReader.java:205)","org.apache.parquet.hadoop.ParquetReader.initReader(ParquetReader.java:170)","org.apache.parquet.hadoop.ParquetReader.read(ParquetReader.java:139)","org.julialang.parquet.n5.ParquetEvidence.readAvro(ParquetEvidence.java:509)","org.julialang.parquet.n5.ParquetEvidence.inspect(ParquetEvidence.java:283)","org.julialang.parquet.n5.OracleMain.generateFixtures(OracleMain.java:258)","org.julialang.parquet.n5.OracleMain.runGenerate(OracleMain.java:121)","org.julialang.parquet.n5.OracleMain.run(OracleMain.java:98)","org.julialang.parquet.n5.OracleMain.main(OracleMain.java:77)"],"exception_chain":[{"class":"java.lang.ClassCastException","message":"repeated int32 element is not a group"}]},"explicit":null}} +{"record":"file","file":"list_rule4_array.v1.parquet","sha256":"4b7a1f524f1f04a6fcb2f8ebabf1f82cf33d709cc154207e43e95ab61d09ed1a","case_id":"list_rule4_array","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule4_array","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule4_array {\n optional group items (LIST) {\n repeated group array {\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":40,"columns":[{"path":"items.array.value","value_count":5,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":40,"total_uncompressed_size":40}]}],"raw_group_rows":["G{items=null}","G{items=G{array=[]}}","G{items=G{array=[G{value=null}]}}","G{items=G{array=[G{value=i32:4},G{value=null}]}}"],"columns":[{"path":"items.array.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":17,"uncompressed_size":17}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":17,"uncompressed_size":17}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule4_array\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"array\",\"fields\":[{\"name\":\"value\",\"type\":[\"null\",\"int\"],\"default\":null}]}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":null}]}","{\"items\":[{\"value\":4},{\"value\":null}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":null}]}","{\"items\":[{\"value\":4},{\"value\":null}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule4_array.v2.parquet","sha256":"966d39ae1c7d1c011b3e2405ef0be9ff9d23372516abcc2060331b61c00bc642","case_id":"list_rule4_array","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule4_array","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule4_array {\n optional group items (LIST) {\n repeated group array {\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":38,"columns":[{"path":"items.array.value","value_count":5,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":38,"total_uncompressed_size":38}]}],"raw_group_rows":["G{items=null}","G{items=G{array=[]}}","G{items=G{array=[G{value=null}]}}","G{items=G{array=[G{value=i32:4},G{value=null}]}}"],"columns":[{"path":"items.array.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":10,"uncompressed_size":10}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":10,"uncompressed_size":10}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule4_array\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"array\",\"fields\":[{\"name\":\"value\",\"type\":[\"null\",\"int\"],\"default\":null}]}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":null}]}","{\"items\":[{\"value\":4},{\"value\":null}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":null}]}","{\"items\":[{\"value\":4},{\"value\":null}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule4_tuple.v1.parquet","sha256":"2b7e4ae67a92182f44cff1c584096bc6b6d741106fe56eb20b0e2d3a41d0a170","case_id":"list_rule4_tuple","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule4_tuple","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule4_tuple {\n optional group items (LIST) {\n repeated group items_tuple {\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":44,"columns":[{"path":"items.items_tuple.value","value_count":5,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":44,"total_uncompressed_size":44}]}],"raw_group_rows":["G{items=null}","G{items=G{items_tuple=[]}}","G{items=G{items_tuple=[G{value=i32:7}]}}","G{items=G{items_tuple=[G{value=null},G{value=i32:8}]}}"],"columns":[{"path":"items.items_tuple.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,3,2,3],"dense":["7","8"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":21,"uncompressed_size":21}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,3,2,3],"dense":["7","8"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":21,"uncompressed_size":21}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule4_tuple\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"items_tuple\",\"fields\":[{\"name\":\"value\",\"type\":[\"null\",\"int\"],\"default\":null}]}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":7}]}","{\"items\":[{\"value\":null},{\"value\":8}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":7}]}","{\"items\":[{\"value\":null},{\"value\":8}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule4_tuple.v2.parquet","sha256":"9d51d23da84416b0b520e0fdb80796d80923b5c8cbdeeea454a8c422a897934a","case_id":"list_rule4_tuple","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule4_tuple","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule4_tuple {\n optional group items (LIST) {\n repeated group items_tuple {\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":43,"columns":[{"path":"items.items_tuple.value","value_count":5,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{items=null}","G{items=G{items_tuple=[]}}","G{items=G{items_tuple=[G{value=i32:7}]}}","G{items=G{items_tuple=[G{value=null},G{value=i32:8}]}}"],"columns":[{"path":"items.items_tuple.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,3,2,3],"dense":["7","8"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,3,2,3],"dense":["7","8"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule4_tuple\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"items_tuple\",\"fields\":[{\"name\":\"value\",\"type\":[\"null\",\"int\"],\"default\":null}]}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":7}]}","{\"items\":[{\"value\":null},{\"value\":8}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{\"value\":7}]}","{\"items\":[{\"value\":null},{\"value\":8}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule5_required.v1.parquet","sha256":"e16a41a44e974c44f8ca83021ecd55e016018bab7f86a930b2560af7d66de762","case_id":"list_rule5_required","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule5_required","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule5_required {\n optional group items (LIST) {\n repeated group list {\n required int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":48,"columns":[{"path":"items.list.element","value_count":5,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":48,"total_uncompressed_size":48}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=i32:10}]}}","G{items=G{list=[G{element=i32:20},G{element=i32:30}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":25,"uncompressed_size":25}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":25,"uncompressed_size":25}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule5_required\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule5_required.v2.parquet","sha256":"a730dbdfca94503ce0f1e6717f96c4636378b2b7f56d593bebae86fa86513127","case_id":"list_rule5_required","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule5_required","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule5_required {\n optional group items (LIST) {\n repeated group list {\n required int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":43,"columns":[{"path":"items.list.element","value_count":5,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=i32:10}]}}","G{items=G{list=[G{element=i32:20},G{element=i32:30}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":2,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,2,2],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":2,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule5_required\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"int\"}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[10]}","{\"items\":[20,30]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule5_optional_paired.v1.parquet","sha256":"2873a69f47bd82a93625a78fda96a58a5783c5f473f0ace861216df876c9a725","case_id":"list_rule5_optional_paired","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule5_optional_paired","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule5_optional_paired {\n optional group items (LIST) {\n repeated group list {\n optional int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":40,"columns":[{"path":"items.list.element","value_count":5,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":40,"total_uncompressed_size":40}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=null}]}}","G{items=G{list=[G{element=i32:4},G{element=null}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":17,"uncompressed_size":17}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":5,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":17,"uncompressed_size":17}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule5_optional_paired\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[4,null]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[4,null]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule5_optional_paired.v2.parquet","sha256":"a465d3f92575429b545d4edad4aece7dd30019860ee71af75f61d035c0b00f04","case_id":"list_rule5_optional_paired","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule5_optional_paired","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule5_optional_paired {\n optional group items (LIST) {\n repeated group list {\n optional int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":38,"columns":[{"path":"items.list.element","value_count":5,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":38,"total_uncompressed_size":38}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=null}]}}","G{items=G{list=[G{element=i32:4},G{element=null}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":10,"uncompressed_size":10}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1],"definition":[0,1,2,3,2],"dense":["4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":5,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":10,"uncompressed_size":10}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule5_optional_paired\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[4,null]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[4,null]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule5_optional_extended.v1.parquet","sha256":"5281d05e2317fbc30e280c18b15078d0676b5ef22b1ee56d9788dc2e45d45af0","case_id":"list_rule5_optional_extended","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule5_optional_extended","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule5_optional_extended {\n optional group items (LIST) {\n repeated group list {\n optional int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":44,"columns":[{"path":"items.list.element","value_count":6,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":44,"total_uncompressed_size":44}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=null}]}}","G{items=G{list=[G{element=i32:5},G{element=null},G{element=i32:6}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1],"definition":[0,1,2,3,2,3],"dense":["5","6"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":6,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":21,"uncompressed_size":21}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1,1],"definition":[0,1,2,3,2,3],"dense":["5","6"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":6,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":21,"uncompressed_size":21}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule5_optional_extended\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[5,null,6]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[5,null,6]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_rule5_optional_extended.v2.parquet","sha256":"e2b4bca7e67bac4dafdca1a4aea01d3bdf764a6ece37419edd2c39699efec6c4","case_id":"list_rule5_optional_extended","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_rule5_optional_extended","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_rule5_optional_extended {\n optional group items (LIST) {\n repeated group list {\n optional int32 element;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":43,"columns":[{"path":"items.list.element","value_count":6,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{items=null}","G{items=G{list=[]}}","G{items=G{list=[G{element=null}]}}","G{items=G{list=[G{element=i32:5},G{element=null},G{element=i32:6}]}}"],"columns":[{"path":"items.list.element","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1],"definition":[0,1,2,3,2,3],"dense":["5","6"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":6,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,1,1],"definition":[0,1,2,3,2,3],"dense":["5","6"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":6,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_rule5_optional_extended\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[5,null,6]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[null]}","{\"items\":[5,null,6]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_direct_map.v1.parquet","sha256":"c84b4c582a00245acbc6330c80a3437f0007e1c2311adf31025b6952a95999e3","case_id":"list_direct_map","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_direct_map","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_direct_map {\n optional group items (LIST) {\n repeated group map (MAP) {\n repeated group key_value {\n required int32 key;\n required int32 value;\n }\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":98,"columns":[{"path":"items.map.key_value.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":49,"total_uncompressed_size":49},{"path":"items.map.key_value.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":49,"total_uncompressed_size":49}]}],"raw_group_rows":["G{items=null}","G{items=G{map=[]}}","G{items=G{map=[G{key_value=[]}]}}","G{items=G{map=[G{key_value=[G{key=i32:1,value=i32:10},G{key=i32:1,value=i32:20}]},G{key_value=[]},G{key_value=[G{key=i32:2,value=i32:30}]}]}}"],"columns":[{"path":"items.map.key_value.key","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","1","2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","1","2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}]}]},{"path":"items.map.key_value.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"rejected","read_schema":null,"materialized_schema":null,"rows":[],"normalized_rows":[],"error_class":"java.lang.IllegalArgumentException","error_message":"Map key type must be binary (UTF8): required int32 key","error_stack":["org.apache.parquet.avro.AvroSchemaConverter$2.visitMapOrMapKeyValue(AvroSchemaConverter.java:516)","org.apache.parquet.avro.AvroSchemaConverter$2.visit(AvroSchemaConverter.java:497)","org.apache.parquet.schema.LogicalTypeAnnotation$MapLogicalTypeAnnotation.accept(LogicalTypeAnnotation.java:446)","org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:462)","org.apache.parquet.avro.AvroSchemaConverter$2.visit(AvroSchemaConverter.java:475)","org.apache.parquet.schema.LogicalTypeAnnotation$ListLogicalTypeAnnotation.accept(LogicalTypeAnnotation.java:485)","org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:462)","org.apache.parquet.avro.AvroSchemaConverter.convertFields(AvroSchemaConverter.java:365)","org.apache.parquet.avro.AvroSchemaConverter.convert(AvroSchemaConverter.java:354)","org.apache.parquet.avro.AvroReadSupport.prepareForRead(AvroReadSupport.java:179)","org.apache.parquet.hadoop.InternalParquetRecordReader.initialize(InternalParquetRecordReader.java:205)","org.apache.parquet.hadoop.ParquetReader.initReader(ParquetReader.java:170)","org.apache.parquet.hadoop.ParquetReader.read(ParquetReader.java:139)","org.julialang.parquet.n5.ParquetEvidence.readAvro(ParquetEvidence.java:509)","org.julialang.parquet.n5.ParquetEvidence.inspect(ParquetEvidence.java:283)","org.julialang.parquet.n5.OracleMain.generateFixtures(OracleMain.java:258)","org.julialang.parquet.n5.OracleMain.runGenerate(OracleMain.java:121)","org.julialang.parquet.n5.OracleMain.run(OracleMain.java:98)","org.julialang.parquet.n5.OracleMain.main(OracleMain.java:77)"],"exception_chain":[{"class":"java.lang.IllegalArgumentException","message":"Map key type must be binary (UTF8): required int32 key"}]},"explicit":null}} +{"record":"file","file":"list_direct_map.v2.parquet","sha256":"c2d8ac178692160961f93b2a572a0bd4b19f71c63eddb56f7585c3e5a30c13eb","case_id":"list_direct_map","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_direct_map","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_direct_map {\n optional group items (LIST) {\n repeated group map (MAP) {\n repeated group key_value {\n required int32 key;\n required int32 value;\n }\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":92,"columns":[{"path":"items.map.key_value.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":48,"total_uncompressed_size":48},{"path":"items.map.key_value.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":44,"total_uncompressed_size":44}]}],"raw_group_rows":["G{items=null}","G{items=G{map=[]}}","G{items=G{map=[G{key_value=[]}]}}","G{items=G{map=[G{key_value=[G{key=i32:1,value=i32:10},G{key=i32:1,value=i32:20}]},G{key_value=[]},G{key_value=[G{key=i32:2,value=i32:30}]}]}}"],"columns":[{"path":"items.map.key_value.key","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","1","2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":20,"uncompressed_size":20}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["1","1","2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":20,"uncompressed_size":20}]}]},{"path":"items.map.key_value.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"rejected","read_schema":null,"materialized_schema":null,"rows":[],"normalized_rows":[],"error_class":"java.lang.IllegalArgumentException","error_message":"Map key type must be binary (UTF8): required int32 key","error_stack":["org.apache.parquet.avro.AvroSchemaConverter$2.visitMapOrMapKeyValue(AvroSchemaConverter.java:516)","org.apache.parquet.avro.AvroSchemaConverter$2.visit(AvroSchemaConverter.java:497)","org.apache.parquet.schema.LogicalTypeAnnotation$MapLogicalTypeAnnotation.accept(LogicalTypeAnnotation.java:446)","org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:462)","org.apache.parquet.avro.AvroSchemaConverter$2.visit(AvroSchemaConverter.java:475)","org.apache.parquet.schema.LogicalTypeAnnotation$ListLogicalTypeAnnotation.accept(LogicalTypeAnnotation.java:485)","org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:462)","org.apache.parquet.avro.AvroSchemaConverter.convertFields(AvroSchemaConverter.java:365)","org.apache.parquet.avro.AvroSchemaConverter.convert(AvroSchemaConverter.java:354)","org.apache.parquet.avro.AvroReadSupport.prepareForRead(AvroReadSupport.java:179)","org.apache.parquet.hadoop.InternalParquetRecordReader.initialize(InternalParquetRecordReader.java:205)","org.apache.parquet.hadoop.ParquetReader.initReader(ParquetReader.java:170)","org.apache.parquet.hadoop.ParquetReader.read(ParquetReader.java:139)","org.julialang.parquet.n5.ParquetEvidence.readAvro(ParquetEvidence.java:509)","org.julialang.parquet.n5.ParquetEvidence.inspect(ParquetEvidence.java:283)","org.julialang.parquet.n5.OracleMain.generateFixtures(OracleMain.java:258)","org.julialang.parquet.n5.OracleMain.runGenerate(OracleMain.java:121)","org.julialang.parquet.n5.OracleMain.run(OracleMain.java:98)","org.julialang.parquet.n5.OracleMain.main(OracleMain.java:77)"],"exception_chain":[{"class":"java.lang.IllegalArgumentException","message":"Map key type must be binary (UTF8): required int32 key"}]},"explicit":null}} +{"record":"file","file":"list_direct_map_utf8.v1.parquet","sha256":"4f7faf3e2bef6c043d15e88feeb796728df51a9648abb81a301e0f6e6f97fd0b","case_id":"list_direct_map_utf8","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_direct_map_utf8","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_direct_map_utf8 {\n optional group items (LIST) {\n repeated group map (MAP) {\n repeated group key_value {\n required binary key (STRING);\n required int32 value;\n }\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":101,"columns":[{"path":"items.map.key_value.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":52,"total_uncompressed_size":52},{"path":"items.map.key_value.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":49,"total_uncompressed_size":49}]}],"raw_group_rows":["G{items=null}","G{items=G{map=[]}}","G{items=G{map=[G{key_value=[]}]}}","G{items=G{map=[G{key_value=[G{key=utf8:a,value=i32:10},G{key=utf8:a,value=i32:20}]},G{key_value=[]},G{key_value=[G{key=utf8:b,value=i32:30}]}]}}"],"columns":[{"path":"items.map.key_value.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["utf8:a","utf8:a","utf8:b"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["utf8:a","utf8:a","utf8:b"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}]}]},{"path":"items.map.key_value.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":4,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":26,"uncompressed_size":26}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_direct_map_utf8\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"map\",\"values\":\"int\"}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{}]}","{\"items\":[{\"a\":20},{},{\"b\":30}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{}]}","{\"items\":[{\"a\":20},{},{\"b\":30}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"list_direct_map_utf8.v2.parquet","sha256":"edc66d338474404c9c7db8a4b93d5f1def987b4883f5aefe7d042c348aa282de","case_id":"list_direct_map_utf8","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":4,"metadata":{"parquet.jl.n5.case_id":"list_direct_map_utf8","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message list_direct_map_utf8 {\n optional group items (LIST) {\n repeated group map (MAP) {\n repeated group key_value {\n required binary key (STRING);\n required int32 value;\n }\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":4,"total_byte_size":116,"columns":[{"path":"items.map.key_value.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY"],"total_compressed_size":72,"total_uncompressed_size":72},{"path":"items.map.key_value.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":44,"total_uncompressed_size":44}]}],"raw_group_rows":["G{items=null}","G{items=G{map=[]}}","G{items=G{map=[G{key_value=[]}]}}","G{items=G{map=[G{key_value=[G{key=utf8:a,value=i32:10},G{key=utf8:a,value=i32:20}]},G{key_value=[]},G{key_value=[G{key=utf8:b,value=i32:30}]}]}}"],"columns":[{"path":"items.map.key_value.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["utf8:a","utf8:a","utf8:b"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["utf8:a","utf8:a","utf8:b"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}]}]},{"path":"items.map.key_value.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":2,"max_definition_level":3,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}],"row_groups":[{"row_group":0,"rows":4,"repetition":[0,0,0,0,2,1,1],"definition":[0,1,2,3,3,2,3],"dense":["10","20","30"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":4,"index_row_count":null,"null_count":4,"encoding":"DELTA_BINARY_PACKED","compressed_size":16,"uncompressed_size":16}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"list_direct_map_utf8\",\"fields\":[{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"map\",\"values\":\"int\"}}],\"default\":null}]}","rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{}]}","{\"items\":[{\"a\":20},{},{\"b\":30}]}"],"normalized_rows":["{\"items\":null}","{\"items\":[]}","{\"items\":[{}]}","{\"items\":[{\"a\":20},{},{\"b\":30}]}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"map_standard.v1.parquet","sha256":"0acccffff97f7ff6867daaa35f57ad1a9609ed3407e81921a275f9c3f40d20d5","case_id":"map_standard","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":5,"metadata":{"parquet.jl.n5.case_id":"map_standard","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_standard {\n optional group map (MAP) {\n repeated group key_value {\n required binary key (STRING);\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":5,"total_byte_size":112,"columns":[{"path":"map.key_value.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":60,"total_uncompressed_size":60},{"path":"map.key_value.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":52,"total_uncompressed_size":52}]}],"raw_group_rows":["G{map=null}","G{map=G{key_value=[]}}","G{map=G{key_value=[G{key=utf8:a,value=null}]}}","G{map=G{key_value=[G{key=utf8:a,value=i32:1},G{key=utf8:a,value=i32:2},G{key=utf8:b,value=i32:3}]}}","G{map=G{key_value=[G{key=utf8:c,value=i32:4}]}}"],"columns":[{"path":"map.key_value.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":38,"uncompressed_size":38}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":38,"uncompressed_size":38}]}]},{"path":"map.key_value.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"map_standard\",\"fields\":[{\"name\":\"map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"normalized_rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"map_standard.v2.parquet","sha256":"136bf8f4949fa97f52b337b7a4c349d6fb2ca2f38e2df815adadea5df134c9fd","case_id":"map_standard","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":5,"metadata":{"parquet.jl.n5.case_id":"map_standard","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_standard {\n optional group map (MAP) {\n repeated group key_value {\n required binary key (STRING);\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":5,"total_byte_size":115,"columns":[{"path":"map.key_value.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY"],"total_compressed_size":72,"total_uncompressed_size":72},{"path":"map.key_value.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{map=null}","G{map=G{key_value=[]}}","G{map=G{key_value=[G{key=utf8:a,value=null}]}}","G{map=G{key_value=[G{key=utf8:a,value=i32:1},G{key=utf8:a,value=i32:2},G{key=utf8:b,value=i32:3}]}}","G{map=G{key_value=[G{key=utf8:c,value=i32:4}]}}"],"columns":[{"path":"map.key_value.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":2,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":2,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}]}]},{"path":"map.key_value.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"map_standard\",\"fields\":[{\"name\":\"map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"normalized_rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"map_arbitrary_names.v1.parquet","sha256":"0a6aadfd449407c0e6036d6b894e4f73c9d6a9b7a64b35eba16da2a685954ccb","case_id":"map_arbitrary_names","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":5,"metadata":{"parquet.jl.n5.case_id":"map_arbitrary_names","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_arbitrary_names {\n optional group bag (MAP) {\n repeated group pairs {\n required binary left (STRING);\n optional int32 right;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":5,"total_byte_size":112,"columns":[{"path":"bag.pairs.left","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":60,"total_uncompressed_size":60},{"path":"bag.pairs.right","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":52,"total_uncompressed_size":52}]}],"raw_group_rows":["G{bag=null}","G{bag=G{pairs=[]}}","G{bag=G{pairs=[G{left=utf8:a,right=null}]}}","G{bag=G{pairs=[G{left=utf8:a,right=i32:1},G{left=utf8:a,right=i32:2},G{left=utf8:b,right=i32:3}]}}","G{bag=G{pairs=[G{left=utf8:c,right=i32:4}]}}"],"columns":[{"path":"bag.pairs.left","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":38,"uncompressed_size":38}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":38,"uncompressed_size":38}]}]},{"path":"bag.pairs.right","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"map_arbitrary_names\",\"fields\":[{\"name\":\"bag\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"bag\":null}","{\"bag\":{}}","{\"bag\":{\"a\":null}}","{\"bag\":{\"a\":2,\"b\":3}}","{\"bag\":{\"c\":4}}"],"normalized_rows":["{\"bag\":null}","{\"bag\":{}}","{\"bag\":{\"a\":null}}","{\"bag\":{\"a\":2,\"b\":3}}","{\"bag\":{\"c\":4}}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"map_arbitrary_names.v2.parquet","sha256":"b69c7d382417e2be0834bbbc060a5a65548867c659aec51283b77650bccd8de4","case_id":"map_arbitrary_names","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":5,"metadata":{"parquet.jl.n5.case_id":"map_arbitrary_names","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_arbitrary_names {\n optional group bag (MAP) {\n repeated group pairs {\n required binary left (STRING);\n optional int32 right;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":5,"total_byte_size":115,"columns":[{"path":"bag.pairs.left","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY"],"total_compressed_size":72,"total_uncompressed_size":72},{"path":"bag.pairs.right","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{bag=null}","G{bag=G{pairs=[]}}","G{bag=G{pairs=[G{left=utf8:a,right=null}]}}","G{bag=G{pairs=[G{left=utf8:a,right=i32:1},G{left=utf8:a,right=i32:2},G{left=utf8:b,right=i32:3}]}}","G{bag=G{pairs=[G{left=utf8:c,right=i32:4}]}}"],"columns":[{"path":"bag.pairs.left","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":2,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":2,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}]}]},{"path":"bag.pairs.right","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"map_arbitrary_names\",\"fields\":[{\"name\":\"bag\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"bag\":null}","{\"bag\":{}}","{\"bag\":{\"a\":null}}","{\"bag\":{\"a\":2,\"b\":3}}","{\"bag\":{\"c\":4}}"],"normalized_rows":["{\"bag\":null}","{\"bag\":{}}","{\"bag\":{\"a\":null}}","{\"bag\":{\"a\":2,\"b\":3}}","{\"bag\":{\"c\":4}}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"map_standalone_mkv.v1.parquet","sha256":"a99a78cc84bdc17a30d0ab73c28c5edf6a7640b85ceab9470fedaa7369a5f401","case_id":"map_standalone_mkv","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":5,"metadata":{"parquet.jl.n5.case_id":"map_standalone_mkv","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_standalone_mkv {\n optional group map (MAP_KEY_VALUE) {\n repeated group entries (MAP_KEY_VALUE) {\n required binary key (STRING);\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":5,"total_byte_size":112,"columns":[{"path":"map.entries.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":60,"total_uncompressed_size":60},{"path":"map.entries.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":52,"total_uncompressed_size":52}]}],"raw_group_rows":["G{map=null}","G{map=G{entries=[]}}","G{map=G{entries=[G{key=utf8:a,value=null}]}}","G{map=G{entries=[G{key=utf8:a,value=i32:1},G{key=utf8:a,value=i32:2},G{key=utf8:b,value=i32:3}]}}","G{map=G{entries=[G{key=utf8:c,value=i32:4}]}}"],"columns":[{"path":"map.entries.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":38,"uncompressed_size":38}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":38,"uncompressed_size":38}]}]},{"path":"map.entries.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":7,"row_count":5,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":29,"uncompressed_size":29}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"map_standalone_mkv\",\"fields\":[{\"name\":\"map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"normalized_rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"map_standalone_mkv.v2.parquet","sha256":"467cdc8abf6514d94aa8698d7b9bcad654684bd5d465cc0b973329a3c9db1f30","case_id":"map_standalone_mkv","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":5,"metadata":{"parquet.jl.n5.case_id":"map_standalone_mkv","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_standalone_mkv {\n optional group map (MAP_KEY_VALUE) {\n repeated group entries (MAP_KEY_VALUE) {\n required binary key (STRING);\n optional int32 value;\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":5,"total_byte_size":115,"columns":[{"path":"map.entries.key","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY"],"total_compressed_size":72,"total_uncompressed_size":72},{"path":"map.entries.value","value_count":7,"codec":"UNCOMPRESSED","encodings":["DELTA_BINARY_PACKED"],"total_compressed_size":43,"total_uncompressed_size":43}]}],"raw_group_rows":["G{map=null}","G{map=G{entries=[]}}","G{map=G{entries=[G{key=utf8:a,value=null}]}}","G{map=G{entries=[G{key=utf8:a,value=i32:1},G{key=utf8:a,value=i32:2},G{key=utf8:b,value=i32:3}]}}","G{map=G{entries=[G{key=utf8:c,value=i32:4}]}}"],"columns":[{"path":"map.entries.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":2,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":2,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,2,2,2,2],"dense":["utf8:a","utf8:a","utf8:a","utf8:b","utf8:c"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":2,"encoding":"DELTA_BYTE_ARRAY","compressed_size":44,"uncompressed_size":44}]}]},{"path":"map.entries.value","physical_type":"INT32","logical_type":null,"type_length":0,"max_repetition_level":1,"max_definition_level":3,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}],"row_groups":[{"row_group":0,"rows":5,"repetition":[0,0,0,0,1,1,0],"definition":[0,1,2,3,3,3,3],"dense":["1","2","3","4"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":7,"row_count":5,"index_row_count":null,"null_count":3,"encoding":"DELTA_BINARY_PACKED","compressed_size":15,"uncompressed_size":15}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"success","read_schema":null,"materialized_schema":"{\"type\":\"record\",\"name\":\"map_standalone_mkv\",\"fields\":[{\"name\":\"map\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"int\"]}],\"default\":null}]}","rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"normalized_rows":["{\"map\":null}","{\"map\":{}}","{\"map\":{\"a\":null}}","{\"map\":{\"a\":2,\"b\":3}}","{\"map\":{\"c\":4}}"],"error_class":null,"error_message":null,"error_stack":[],"exception_chain":[]},"explicit":null}} +{"record":"file","file":"map_key_only.v1.parquet","sha256":"e05e566da94fcdd96d5ca17c4f8f4403541a50c94002bef15639df889fb79102","case_id":"map_key_only","page_version":"v1","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":3,"metadata":{"parquet.jl.n5.case_id":"map_key_only","parquet.jl.n5.page_version":"v1","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_key_only {\n required group map (MAP) {\n repeated group key_value {\n required binary key (STRING);\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":3,"total_byte_size":52,"columns":[{"path":"map.key_value.key","value_count":4,"codec":"UNCOMPRESSED","encodings":["PLAIN","RLE"],"total_compressed_size":52,"total_uncompressed_size":52}]}],"raw_group_rows":["G{map=G{key_value=[]}}","G{map=G{key_value=[G{key=utf8:k1}]}}","G{map=G{key_value=[G{key=utf8:k2},G{key=utf8:k2}]}}"],"columns":[{"path":"map.key_value.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":1,"repetition":[0,0,0,1],"definition":[0,1,1,1],"dense":["utf8:k1","utf8:k2","utf8:k2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":4,"row_count":3,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":30,"uncompressed_size":30}],"row_groups":[{"row_group":0,"rows":3,"repetition":[0,0,0,1],"definition":[0,1,1,1],"dense":["utf8:k1","utf8:k2","utf8:k2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V1","value_count":4,"row_count":3,"index_row_count":null,"null_count":null,"encoding":"PLAIN","compressed_size":30,"uncompressed_size":30}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"rejected","read_schema":null,"materialized_schema":null,"rows":[],"normalized_rows":[],"error_class":"java.lang.UnsupportedOperationException","error_message":"Invalid map type required group map (MAP) { repeated group key_value { required binary key (STRING); } }","error_stack":["org.apache.parquet.avro.AvroSchemaConverter$2.visitMapOrMapKeyValue(AvroSchemaConverter.java:508)","org.apache.parquet.avro.AvroSchemaConverter$2.visit(AvroSchemaConverter.java:497)","org.apache.parquet.schema.LogicalTypeAnnotation$MapLogicalTypeAnnotation.accept(LogicalTypeAnnotation.java:446)","org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:462)","org.apache.parquet.avro.AvroSchemaConverter.convertFields(AvroSchemaConverter.java:365)","org.apache.parquet.avro.AvroSchemaConverter.convert(AvroSchemaConverter.java:354)","org.apache.parquet.avro.AvroReadSupport.prepareForRead(AvroReadSupport.java:179)","org.apache.parquet.hadoop.InternalParquetRecordReader.initialize(InternalParquetRecordReader.java:205)","org.apache.parquet.hadoop.ParquetReader.initReader(ParquetReader.java:170)","org.apache.parquet.hadoop.ParquetReader.read(ParquetReader.java:139)","org.julialang.parquet.n5.ParquetEvidence.readAvro(ParquetEvidence.java:509)","org.julialang.parquet.n5.ParquetEvidence.inspect(ParquetEvidence.java:283)","org.julialang.parquet.n5.OracleMain.generateFixtures(OracleMain.java:258)","org.julialang.parquet.n5.OracleMain.runGenerate(OracleMain.java:121)","org.julialang.parquet.n5.OracleMain.run(OracleMain.java:98)","org.julialang.parquet.n5.OracleMain.main(OracleMain.java:77)"],"exception_chain":[{"class":"java.lang.UnsupportedOperationException","message":"Invalid map type required group map (MAP) { repeated group key_value { required binary key (STRING); } }"}]},"explicit":null}} +{"record":"file","file":"map_key_only.v2.parquet","sha256":"3b6d03072c1046e000097bad8392319678256610a1ad56e7f6611c390cb0a5ef","case_id":"map_key_only","page_version":"v2","created_by":"parquet-mr version 1.17.1 (build 78a8d3230eb4769db93de5f2f2e18363c04cae81)","row_count":3,"metadata":{"parquet.jl.n5.case_id":"map_key_only","parquet.jl.n5.page_version":"v2","parquet.jl.n5.parquet_java_commit":"78a8d3230eb4769db93de5f2f2e18363c04cae81","parquet.jl.n5.producer":"parquet-java","writer.model.name":"example"},"physical_schema":"message map_key_only {\n required group map (MAP) {\n repeated group key_value {\n required binary key (STRING);\n }\n }\n}\n","row_groups":[{"ordinal":0,"row_count":3,"total_byte_size":55,"columns":[{"path":"map.key_value.key","value_count":4,"codec":"UNCOMPRESSED","encodings":["DELTA_BYTE_ARRAY"],"total_compressed_size":55,"total_uncompressed_size":55}]}],"raw_group_rows":["G{map=G{key_value=[]}}","G{map=G{key_value=[G{key=utf8:k1}]}}","G{map=G{key_value=[G{key=utf8:k2},G{key=utf8:k2}]}}"],"columns":[{"path":"map.key_value.key","physical_type":"BINARY","logical_type":"STRING","type_length":0,"max_repetition_level":1,"max_definition_level":1,"repetition":[0,0,0,1],"definition":[0,1,1,1],"dense":["utf8:k1","utf8:k2","utf8:k2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":4,"row_count":3,"index_row_count":null,"null_count":1,"encoding":"DELTA_BYTE_ARRAY","compressed_size":27,"uncompressed_size":27}],"row_groups":[{"row_group":0,"rows":3,"repetition":[0,0,0,1],"definition":[0,1,1,1],"dense":["utf8:k1","utf8:k2","utf8:k2"],"dictionaries":[],"pages":[{"row_group":0,"ordinal":0,"type":"DATA_PAGE_V2","value_count":4,"row_count":3,"index_row_count":null,"null_count":1,"encoding":"DELTA_BYTE_ARRAY","compressed_size":27,"uncompressed_size":27}]}]}],"avro":{"add_list_element_records":false,"inferred":{"status":"rejected","read_schema":null,"materialized_schema":null,"rows":[],"normalized_rows":[],"error_class":"java.lang.UnsupportedOperationException","error_message":"Invalid map type required group map (MAP) { repeated group key_value { required binary key (STRING); } }","error_stack":["org.apache.parquet.avro.AvroSchemaConverter$2.visitMapOrMapKeyValue(AvroSchemaConverter.java:508)","org.apache.parquet.avro.AvroSchemaConverter$2.visit(AvroSchemaConverter.java:497)","org.apache.parquet.schema.LogicalTypeAnnotation$MapLogicalTypeAnnotation.accept(LogicalTypeAnnotation.java:446)","org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:462)","org.apache.parquet.avro.AvroSchemaConverter.convertFields(AvroSchemaConverter.java:365)","org.apache.parquet.avro.AvroSchemaConverter.convert(AvroSchemaConverter.java:354)","org.apache.parquet.avro.AvroReadSupport.prepareForRead(AvroReadSupport.java:179)","org.apache.parquet.hadoop.InternalParquetRecordReader.initialize(InternalParquetRecordReader.java:205)","org.apache.parquet.hadoop.ParquetReader.initReader(ParquetReader.java:170)","org.apache.parquet.hadoop.ParquetReader.read(ParquetReader.java:139)","org.julialang.parquet.n5.ParquetEvidence.readAvro(ParquetEvidence.java:509)","org.julialang.parquet.n5.ParquetEvidence.inspect(ParquetEvidence.java:283)","org.julialang.parquet.n5.OracleMain.generateFixtures(OracleMain.java:258)","org.julialang.parquet.n5.OracleMain.runGenerate(OracleMain.java:121)","org.julialang.parquet.n5.OracleMain.run(OracleMain.java:98)","org.julialang.parquet.n5.OracleMain.main(OracleMain.java:77)"],"exception_chain":[{"class":"java.lang.UnsupportedOperationException","message":"Invalid map type required group map (MAP) { repeated group key_value { required binary key (STRING); } }"}]},"explicit":null}} diff --git a/test/conformance/n5/external-files.sha256 b/test/conformance/n5/external-files.sha256 new file mode 100644 index 0000000..1f1aef7 --- /dev/null +++ b/test/conformance/n5/external-files.sha256 @@ -0,0 +1,42 @@ +e1469a653fc5e8f26db3d111d4af366c90aff24403912c7bc20b8a47b16dd16e test/conformance/n5/expected/arrow-rs-rule3-near-neighbor.json +12f821dac1a2f2187302947c505d2656b9da05d936c5fcdc08ac11e1deff7fba test/conformance/n5/expected/arrow-rs.json +2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671 test/conformance/n5/expected/parquet-java.jsonl +acf5536a16167a2a870dfd55ad2a5d208712069d8e0f034cf396fa6d6c193184 test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v1.parquet +baaf9a09bc397db8c0b1d0453fa2d87e36f6a74bbc56778be39d27304007cbb3 test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v2.parquet +40eb4521da12a9bd5db38cfe9e1f31dd91bd1095ec5b24b94f9a02318e40128f test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet +4fe0319ea46248612489d845eb6fdbfa127077f06a9e7a2efc14fa67def66ce1 test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet +916475c20dde6afe338d60e43d2e4a6293c2bc26efb6dd439d1d7d9299834bc6 test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v1.parquet +ef72761c509f8bc302986d130f6d7d9382b975c8a3462a955c599bd5f427edac test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v2.parquet +a33be66fe1a5e7189c457b49163e5a4b658228ffb5054a40cd22f20f731ba21a test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v1.parquet +f010b919bcc2d4e8c42f6d97df651266de06b6352f6b993756fd0a007e1c5028 test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v2.parquet +c84b4c582a00245acbc6330c80a3437f0007e1c2311adf31025b6952a95999e3 test/conformance/n5/golden/parquet-java/list_direct_map.v1.parquet +c2d8ac178692160961f93b2a572a0bd4b19f71c63eddb56f7585c3e5a30c13eb test/conformance/n5/golden/parquet-java/list_direct_map.v2.parquet +4f7faf3e2bef6c043d15e88feeb796728df51a9648abb81a301e0f6e6f97fd0b test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v1.parquet +edc66d338474404c9c7db8a4b93d5f1def987b4883f5aefe7d042c348aa282de test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v2.parquet +3466fa00347fae5832a8b751cd50d74e0af0df490d7262bdccf36f0c2f38c2db test/conformance/n5/golden/parquet-java/list_rule1_primitive.v1.parquet +056989f7d112e735fc4f4e835278ba58610e522e1aa3a3e05afdaf540c0463cd test/conformance/n5/golden/parquet-java/list_rule1_primitive.v2.parquet +b3c6ddd69b4b75bfe0ba1fcb21aca6db5d7b7c083fda0cd106b7229baa0a2f7b test/conformance/n5/golden/parquet-java/list_rule2_struct.v1.parquet +2b059b3b5b375c736e7d0abb69a5f1e8dc48606008ffd9e3910f34d3543fc6a6 test/conformance/n5/golden/parquet-java/list_rule2_struct.v2.parquet +c85821a7f6593ad3092f2efd3c8acc602f4969eb8dc36d49f1868b22f1d7d56a test/conformance/n5/golden/parquet-java/list_rule3_nested.v1.parquet +5333c37c239ae57781c63eec972537eb428e275da8d3cca0634acb963989e73e test/conformance/n5/golden/parquet-java/list_rule3_nested.v2.parquet +ab2c1a962ee992845540c6aa04056f15cdc4b40cf89627fb5160dd3fb07a5123 test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet +51ece6eccb1123e817c7e4a9f5f0ff235299501c7e7aa4e78f2878ecb4ae4e5f test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet +4b7a1f524f1f04a6fcb2f8ebabf1f82cf33d709cc154207e43e95ab61d09ed1a test/conformance/n5/golden/parquet-java/list_rule4_array.v1.parquet +966d39ae1c7d1c011b3e2405ef0be9ff9d23372516abcc2060331b61c00bc642 test/conformance/n5/golden/parquet-java/list_rule4_array.v2.parquet +2b7e4ae67a92182f44cff1c584096bc6b6d741106fe56eb20b0e2d3a41d0a170 test/conformance/n5/golden/parquet-java/list_rule4_tuple.v1.parquet +9d51d23da84416b0b520e0fdb80796d80923b5c8cbdeeea454a8c422a897934a test/conformance/n5/golden/parquet-java/list_rule4_tuple.v2.parquet +5281d05e2317fbc30e280c18b15078d0676b5ef22b1ee56d9788dc2e45d45af0 test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v1.parquet +e2b4bca7e67bac4dafdca1a4aea01d3bdf764a6ece37419edd2c39699efec6c4 test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v2.parquet +2873a69f47bd82a93625a78fda96a58a5783c5f473f0ace861216df876c9a725 test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v1.parquet +a465d3f92575429b545d4edad4aece7dd30019860ee71af75f61d035c0b00f04 test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v2.parquet +e16a41a44e974c44f8ca83021ecd55e016018bab7f86a930b2560af7d66de762 test/conformance/n5/golden/parquet-java/list_rule5_required.v1.parquet +a730dbdfca94503ce0f1e6717f96c4636378b2b7f56d593bebae86fa86513127 test/conformance/n5/golden/parquet-java/list_rule5_required.v2.parquet +0a6aadfd449407c0e6036d6b894e4f73c9d6a9b7a64b35eba16da2a685954ccb test/conformance/n5/golden/parquet-java/map_arbitrary_names.v1.parquet +b69c7d382417e2be0834bbbc060a5a65548867c659aec51283b77650bccd8de4 test/conformance/n5/golden/parquet-java/map_arbitrary_names.v2.parquet +e05e566da94fcdd96d5ca17c4f8f4403541a50c94002bef15639df889fb79102 test/conformance/n5/golden/parquet-java/map_key_only.v1.parquet +3b6d03072c1046e000097bad8392319678256610a1ad56e7f6611c390cb0a5ef test/conformance/n5/golden/parquet-java/map_key_only.v2.parquet +a99a78cc84bdc17a30d0ab73c28c5edf6a7640b85ceab9470fedaa7369a5f401 test/conformance/n5/golden/parquet-java/map_standalone_mkv.v1.parquet +467cdc8abf6514d94aa8698d7b9bcad654684bd5d465cc0b973329a3c9db1f30 test/conformance/n5/golden/parquet-java/map_standalone_mkv.v2.parquet +0acccffff97f7ff6867daaa35f57ad1a9609ed3407e81921a275f9c3f40d20d5 test/conformance/n5/golden/parquet-java/map_standard.v1.parquet +136bf8f4949fa97f52b337b7a4c349d6fb2ca2f38e2df815adadea5df134c9fd test/conformance/n5/golden/parquet-java/map_standard.v2.parquet +794a21863f082a15988c92f99a126d815f50f290f435f6481e641323c93a5296 test/conformance/n5/manifest.toml diff --git a/test/conformance/n5/external.jl b/test/conformance/n5/external.jl new file mode 100644 index 0000000..8c29365 --- /dev/null +++ b/test/conformance/n5/external.jl @@ -0,0 +1,659 @@ +using SHA +using TOML +using Test + +const N5_EXTERNAL_ROOT = @__DIR__ + +const N5_EXTERNAL_PRODUCER_PINS = Dict( + "parquet-java" => ( + version="1.17.1", + commit="78a8d3230eb4769db93de5f2f2e18363c04cae81", + ), + "arrow-rs" => ( + version="59.2.0", + commit="782e5a685501a9db6cc8e9a3b7cbff894940c47a", + ), +) + +struct N5ExternalEvidence + producer::String + producer_version::String + producer_commit::String + scope::String + file::String + sha256::String + fixture_count::Int +end + +struct N5ExternalFixture + producer::String + producer_version::String + producer_commit::String + case_id::String + role::String + page_version::Symbol + file::String + sha256::String + rows::Int + evidence_file::String + evidence_sha256::String +end + +struct N5ExternalManifest + evidence::Vector{N5ExternalEvidence} + fixtures::Vector{N5ExternalFixture} +end + +function n5externalrequirekeys(value, required::Vector{String}, label::String) + value isa AbstractDict || throw(ArgumentError("$label must be a TOML table")) + actual = Set(String(key) for key in keys(value)) + expected = Set(required) + actual == expected && return value + missing = sort!(collect(setdiff(expected, actual))) + extra = sort!(collect(setdiff(actual, expected))) + throw(ArgumentError("$label keys differ: missing=$missing extra=$extra")) +end + +function n5externalstring(value, label::String) + value isa String || throw(ArgumentError("$label must be a string")) + isempty(value) && throw(ArgumentError("$label must not be empty")) + return value +end + +function n5externalinteger(value, label::String) + value isa Integer || throw(ArgumentError("$label must be an integer")) + 0 <= value <= typemax(Int) || throw(ArgumentError("$label is outside the Int range")) + return Int(value) +end + +function n5externalsha256(value, label::String) + digest = n5externalstring(value, label) + occursin(r"^[0-9a-f]{64}$", digest) || throw(ArgumentError( + "$label must be a lowercase SHA-256 digest")) + return digest +end + +function n5externalgitcommit(value, label::String) + commit = n5externalstring(value, label) + occursin(r"^[0-9a-f]{40}$", commit) || throw(ArgumentError( + "$label must be a lowercase 40-hex Git commit")) + return commit +end + +function n5externalrelativepath(value, label::String) + path = n5externalstring(value, label) + isabspath(path) && throw(ArgumentError("$label must be relative")) + occursin('\\', path) && throw(ArgumentError("$label must use forward slashes")) + parts = split(path, '/') + any(part -> isempty(part) || part in (".", ".."), parts) && + throw(ArgumentError("$label is not a normalized relative path")) + # normpath emits the platform separator, so compare in the manifest's own + # forward-slash form to keep this check meaningful on Windows. + replace(normpath(path), '\\' => '/') == path || throw(ArgumentError( + "$label is not a normalized relative path")) + return path +end + +function n5externalevidence(value, index::Int) + label = "expected_evidence[$index]" + table = n5externalrequirekeys(value, [ + "producer", + "producer_version", + "producer_commit", + "scope", + "file", + "sha256", + "fixture_count", + ], label) + producer = n5externalstring(table["producer"], "$label.producer") + pin = get(N5_EXTERNAL_PRODUCER_PINS, producer, nothing) + pin === nothing && throw(ArgumentError("$label has unknown producer $producer")) + version = n5externalstring(table["producer_version"], + "$label.producer_version") + commit = n5externalgitcommit(table["producer_commit"], + "$label.producer_commit") + version == pin.version || throw(ArgumentError( + "$label producer version differs from the checked pin")) + commit == pin.commit || throw(ArgumentError( + "$label producer commit differs from the checked pin")) + scope = n5externalstring(table["scope"], "$label.scope") + scope in ("owned", "near-neighbor") || throw(ArgumentError( + "$label has unsupported scope $scope")) + file = n5externalrelativepath(table["file"], "$label.file") + startswith(file, "expected/") || throw(ArgumentError( + "$label file must be below expected/")) + digest = n5externalsha256(table["sha256"], "$label.sha256") + count = n5externalinteger(table["fixture_count"], "$label.fixture_count") + count > 0 || throw(ArgumentError("$label.fixture_count must be positive")) + return N5ExternalEvidence(producer, version, commit, scope, file, + digest, count) +end + +function n5externalcase(value, index::Int) + label = "case[$index]" + table = n5externalrequirekeys(value, [ + "producer", + "producer_version", + "producer_commit", + "case_id", + "role", + "rows", + "expected_evidence_file", + "expected_evidence_sha256", + "files", + ], label) + producer = n5externalstring(table["producer"], "$label.producer") + pin = get(N5_EXTERNAL_PRODUCER_PINS, producer, nothing) + pin === nothing && throw(ArgumentError("$label has unknown producer $producer")) + version = n5externalstring(table["producer_version"], + "$label.producer_version") + commit = n5externalgitcommit(table["producer_commit"], + "$label.producer_commit") + version == pin.version || throw(ArgumentError( + "$label producer version differs from the checked pin")) + commit == pin.commit || throw(ArgumentError( + "$label producer commit differs from the checked pin")) + case_id = n5externalstring(table["case_id"], "$label.case_id") + role = n5externalstring(table["role"], "$label.role") + role in ("binding", "compatibility", "diagnostic") || + throw(ArgumentError("$label has unsupported role $role")) + rows = n5externalinteger(table["rows"], "$label.rows") + evidence_file = n5externalrelativepath(table["expected_evidence_file"], + "$label.expected_evidence_file") + evidence_sha256 = n5externalsha256( + table["expected_evidence_sha256"], + "$label.expected_evidence_sha256") + rawfiles = table["files"] + rawfiles isa AbstractVector || throw(ArgumentError( + "$label.files must be an array of inline tables")) + length(rawfiles) == 2 || throw(ArgumentError( + "$label.files must contain exactly V1 and V2")) + fixtures = N5ExternalFixture[] + sizehint!(fixtures, 2) + pages = Set{Symbol}() + for (fileindex, rawfile) in enumerate(rawfiles) + filelabel = "$label.files[$fileindex]" + filetable = n5externalrequirekeys(rawfile, + ["page_version", "file", "sha256"], filelabel) + pagestring = n5externalstring(filetable["page_version"], + "$filelabel.page_version") + pagestring in ("v1", "v2") || throw(ArgumentError( + "$filelabel has unsupported page version $pagestring")) + page = Symbol(pagestring) + page in pages && throw(ArgumentError( + "$label contains duplicate page version $page")) + push!(pages, page) + file = n5externalrelativepath(filetable["file"], "$filelabel.file") + startswith(file, "golden/$producer/") || throw(ArgumentError( + "$filelabel is outside the producer fixture directory")) + endswith(file, ".parquet") || throw(ArgumentError( + "$filelabel is not a Parquet file")) + digest = n5externalsha256(filetable["sha256"], "$filelabel.sha256") + push!(fixtures, N5ExternalFixture(producer, version, commit, + case_id, role, page, file, digest, rows, evidence_file, + evidence_sha256)) + end + pages == Set((:v1, :v2)) || throw(ArgumentError( + "$label does not bind both page versions")) + return fixtures +end + +function n5externalloadmanifest(path::String) + raw = TOML.parsefile(path) + n5externalrequirekeys(raw, + ["manifest_version", "expected_evidence", "case"], "manifest") + n5externalinteger(raw["manifest_version"], "manifest_version") == 1 || + throw(ArgumentError("unsupported N5 external manifest version")) + rawevidence = raw["expected_evidence"] + rawevidence isa AbstractVector || throw(ArgumentError( + "expected_evidence must be an array of tables")) + evidence = N5ExternalEvidence[ + n5externalevidence(value, index) + for (index, value) in enumerate(rawevidence) + ] + length(unique(item.file for item in evidence)) == length(evidence) || + throw(ArgumentError("expected evidence files are duplicated")) + rawcases = raw["case"] + rawcases isa AbstractVector || throw(ArgumentError( + "case must be an array of tables")) + fixtures = N5ExternalFixture[] + for (index, value) in enumerate(rawcases) + append!(fixtures, n5externalcase(value, index)) + end + identities = [(fixture.producer, fixture.case_id, + fixture.page_version) for fixture in fixtures] + length(unique(identities)) == length(identities) || throw(ArgumentError( + "external fixture producer/case/page identities are duplicated")) + length(unique(fixture.file for fixture in fixtures)) == length(fixtures) || + throw(ArgumentError("external fixture files are duplicated")) + for fixture in fixtures + matches = [item for item in evidence if + item.file == fixture.evidence_file && + item.sha256 == fixture.evidence_sha256] + length(matches) == 1 || throw(ArgumentError( + "$(fixture.producer)/$(fixture.case_id) has no unique " * + "cryptographic evidence reference")) + item = only(matches) + item.producer == fixture.producer || throw(ArgumentError( + "$(fixture.producer)/$(fixture.case_id) evidence producer differs")) + item.producer_version == fixture.producer_version || + throw(ArgumentError("$(fixture.producer)/$(fixture.case_id) evidence version differs")) + item.producer_commit == fixture.producer_commit || + throw(ArgumentError("$(fixture.producer)/$(fixture.case_id) evidence commit differs")) + end + for item in evidence + boundcount = count(fixture -> + fixture.evidence_file == item.file && + fixture.evidence_sha256 == item.sha256, fixtures) + boundcount == item.fixture_count || throw(ArgumentError( + "$(item.file) binds $boundcount fixtures, expected $(item.fixture_count)")) + end + return N5ExternalManifest(evidence, fixtures) +end + +function n5externallist(values...) + return (:list, Any[values...]) +end + +function n5externalstruct(values::Pair...) + output = Pair{String,Any}[] + sizehint!(output, length(values)) + for value in values + push!(output, Pair{String,Any}(String(value.first), value.second)) + end + return (:struct, output) +end + +function n5externalmap(values::Pair...) + output = Pair{Any,Any}[] + sizehint!(output, length(values)) + for value in values + push!(output, Pair{Any,Any}(value.first, value.second)) + end + return (:map, output) +end + +function n5externalrows(name::String, values...) + return [Pair{String,Any}[Pair{String,Any}(name, value)] for value in values] +end + +function n5externalexpectedrows() + required = n5externalrows("items", + missing, + n5externallist(), + n5externallist(Int32(10)), + n5externallist(Int32(20), Int32(30))) + nesteditems = n5externalrows("items", + missing, + n5externallist(), + n5externallist(n5externallist()), + n5externallist(n5externallist(Int32(1), Int32(2)), + n5externallist(), n5externallist(Int32(3)))) + nestedvalues = n5externalrows("values", + missing, + n5externallist(), + n5externallist(n5externallist()), + n5externallist(n5externallist(Int32(1), Int32(2)), + n5externallist(), n5externallist(Int32(3)))) + unannotateditems = n5externalrows("items", + missing, + n5externallist(), + n5externallist(n5externalstruct("element" => n5externallist())), + n5externallist( + n5externalstruct("element" => n5externallist(Int32(1), Int32(2))), + n5externalstruct("element" => n5externallist()), + n5externalstruct("element" => n5externallist(Int32(3))))) + unannotatedvalues = n5externalrows("values", + missing, + n5externallist(), + n5externallist(n5externalstruct("element" => n5externallist())), + n5externallist( + n5externalstruct("element" => n5externallist(Int32(1), Int32(2))), + n5externalstruct("element" => n5externallist()), + n5externalstruct("element" => n5externallist(Int32(3))))) + standardmap = n5externalrows("map", + missing, + n5externalmap(), + n5externalmap("a" => missing), + n5externalmap("a" => Int32(1), "a" => Int32(2), "b" => Int32(3)), + n5externalmap("c" => Int32(4))) + output = Dict{String,Vector{Vector{Pair{String,Any}}}}() + output["parquet-java/list_rule1_primitive"] = required + output["parquet-java/list_rule5_required"] = required + output["parquet-java/list_rule2_struct"] = n5externalrows("items", + missing, + n5externallist(), + n5externallist(n5externalstruct( + "x" => Int32(1), "y" => missing)), + n5externallist( + n5externalstruct("x" => Int32(2), "y" => Int32(20)), + n5externalstruct("x" => Int32(3), "y" => Int32(30)))) + output["parquet-java/list_rule3_nested"] = nesteditems + output["parquet-java/list_rule3_unannotated_diagnostic"] = + unannotateditems + output["parquet-java/list_rule4_array"] = n5externalrows("items", + missing, + n5externallist(), + n5externallist(n5externalstruct("value" => missing)), + n5externallist(n5externalstruct("value" => Int32(4)), + n5externalstruct("value" => missing))) + output["parquet-java/list_rule4_tuple"] = n5externalrows("items", + missing, + n5externallist(), + n5externallist(n5externalstruct("value" => Int32(7))), + n5externallist(n5externalstruct("value" => missing), + n5externalstruct("value" => Int32(8)))) + output["parquet-java/list_rule5_optional_paired"] = n5externalrows("items", + missing, + n5externallist(), + n5externallist(missing), + n5externallist(Int32(4), missing)) + output["parquet-java/list_rule5_optional_extended"] = n5externalrows("items", + missing, + n5externallist(), + n5externallist(missing), + n5externallist(Int32(5), missing, Int32(6))) + output["parquet-java/list_direct_map"] = n5externalrows("items", + missing, + n5externallist(), + n5externallist(n5externalmap()), + n5externallist( + n5externalmap(Int32(1) => Int32(10), Int32(1) => Int32(20)), + n5externalmap(), n5externalmap(Int32(2) => Int32(30)))) + output["parquet-java/list_direct_map_utf8"] = n5externalrows("items", + missing, + n5externallist(), + n5externallist(n5externalmap()), + n5externallist( + n5externalmap("a" => Int32(10), "a" => Int32(20)), + n5externalmap(), n5externalmap("b" => Int32(30)))) + output["parquet-java/map_standard"] = standardmap + output["parquet-java/map_standalone_mkv"] = standardmap + output["parquet-java/map_arbitrary_names"] = n5externalrows("bag", + missing, + n5externalmap(), + n5externalmap("a" => missing), + n5externalmap("a" => Int32(1), "a" => Int32(2), "b" => Int32(3)), + n5externalmap("c" => Int32(4))) + output["parquet-java/map_key_only"] = n5externalrows("map", + n5externalmap(), + n5externalmap("k1" => missing), + n5externalmap("k2" => missing, "k2" => missing)) + output["arrow-rs/arrow-rs-duplicate-keys"] = n5externalrows("entries", + missing, + n5externalmap(), + n5externalmap("a" => missing), + n5externalmap("a" => Int32(1), "a" => Int32(2), "b" => Int32(3)), + n5externalmap("c" => Int32(4))) + output["arrow-rs/arrow-rs-optional-key-present"] = n5externalrows("entries", + missing, + n5externalmap(), + n5externalmap("a" => Int32(1)), + n5externalmap("b" => Int32(2), "c" => Int32(3))) + output["arrow-rs/arrow-rs-list-rule3"] = nestedvalues + output["arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor"] = + unannotatedvalues + return output +end + +const N5_EXTERNAL_EXPECTED_ROWS = n5externalexpectedrows() + +function n5externalnormalize(value) + value === missing && return missing + if value isa Parquet.StructValue + output = Pair{String,Any}[] + sizehint!(output, length(value)) + for pair in value + push!(output, Pair{String,Any}(String(pair.first), + n5externalnormalize(pair.second))) + end + return (:struct, output) + elseif value isa Parquet.MapValue + output = Pair{Any,Any}[] + sizehint!(output, length(value)) + for pair in value + push!(output, Pair{Any,Any}(n5externalnormalize(pair.first), + n5externalnormalize(pair.second))) + end + return (:map, output) + elseif value isa Parquet.ListValue + return (:list, Any[n5externalnormalize(item) for item in value]) + elseif value isa AbstractVector{UInt8} + return (:bytes, bytes2hex(value)) + elseif value isa AbstractVector + throw(ArgumentError("unexpected materialized vector $(typeof(value))")) + end + return value +end + +function n5externalnormalizerows(table::Parquet.Table) + columns = collect(pairs(table.columns)) + output = Vector{Vector{Pair{String,Any}}}(undef, table.rows) + for row in 1:table.rows + values = Pair{String,Any}[] + sizehint!(values, length(columns)) + for column in columns + push!(values, Pair{String,Any}(String(column.first), + n5externalnormalize(column.second[row]))) + end + output[row] = values + end + return output +end + +function n5externalpageversions(table::Parquet.Table) + versions = Symbol[] + for group in table.metadata.row_groups + for column in group.columns + metadata = column.meta_data + metadata === nothing && throw(ArgumentError( + "external column chunk has no metadata")) + firstbyte, stop = Parquet._chunkrange(metadata, + table.file.footer.offset) + position = firstbyte + while position < stop + frame = Parquet.readpage(table.file.source, position, stop, + Parquet.Limits()) + kind = Parquet.pagekind(frame) + if kind === :data_v1 + push!(versions, :v1) + elseif kind === :data_v2 + push!(versions, :v2) + else + throw(ArgumentError("external fixture has $kind page")) + end + position = Parquet.pageend(frame) + end + position == stop || throw(ArgumentError( + "external page scan did not end at the column boundary")) + end + end + isempty(versions) && throw(ArgumentError("external fixture has no data pages")) + return versions +end + +function n5externalfiles(root::String) + output = String[] + for (directory, _, files) in walkdir(root) + for file in files + path = relpath(joinpath(directory, file), N5_EXTERNAL_ROOT) + push!(output, replace(path, '\\' => '/')) + end + end + sort!(output) + return output +end + +function n5externalexactset(label::String, declared, actual) + declaredset = Set(declared) + actualset = Set(actual) + declaredset == actualset && return + missing = sort!(collect(setdiff(declaredset, actualset))) + extra = sort!(collect(setdiff(actualset, declaredset))) + throw(ArgumentError("$label differs: missing=$missing extra=$extra")) +end + +function n5externalfilehash(path::String) + return open(path, "r") do input + return bytes2hex(SHA.sha256(input)) + end +end + +function n5externalcheckhash(relative::String, expected::String) + path = joinpath(N5_EXTERNAL_ROOT, split(relative, '/')...) + isfile(path) || throw(ArgumentError("required N5 external file is absent: $relative")) + actual = n5externalfilehash(path) + actual == expected || throw(ArgumentError( + "$relative SHA-256 differs: expected $expected, got $actual")) + return path +end + +function n5externalrewrite(table::Parquet.Table, pageversion::Symbol) + return N5.n5productionencodedbytes(table, pageversion) +end + +function n5externaltableerror(bytes::Vector{UInt8}) + table = try + Parquet.Table(bytes) + catch error + return error + end + close(table) + return nothing +end + +const N5_EXTERNAL_MANIFEST_PATH = joinpath(N5_EXTERNAL_ROOT, "manifest.toml") +const N5_EXTERNAL_MANIFEST = n5externalloadmanifest(N5_EXTERNAL_MANIFEST_PATH) + +@testset "N5 external fixture manifest" begin + manifest = N5_EXTERNAL_MANIFEST + @test length(manifest.evidence) == 3 + @test length(manifest.fixtures) == 38 + @test count(fixture -> fixture.producer == "parquet-java", + manifest.fixtures) == 30 + @test count(fixture -> fixture.producer == "arrow-rs", + manifest.fixtures) == 8 + manifestcases = Set("$(fixture.producer)/$(fixture.case_id)" + for fixture in manifest.fixtures) + n5externalexactset("external semantic case set", + keys(N5_EXTERNAL_EXPECTED_ROWS), manifestcases) + n5externalexactset("external fixture file set", + (fixture.file for fixture in manifest.fixtures), + n5externalfiles(joinpath(N5_EXTERNAL_ROOT, "golden"))) + n5externalexactset("external expected-evidence file set", + (item.file for item in manifest.evidence), + n5externalfiles(joinpath(N5_EXTERNAL_ROOT, "expected"))) + for item in manifest.evidence + n5externalcheckhash(item.file, item.sha256) + end + for fixture in manifest.fixtures + n5externalcheckhash(fixture.file, fixture.sha256) + end + @test_throws ArgumentError n5externalexactset("synthetic missing", + ["a", "b"], ["a"]) + @test_throws ArgumentError n5externalexactset("synthetic extra", + ["a"], ["a", "b"]) + with_extra = TOML.parsefile(N5_EXTERNAL_MANIFEST_PATH) + with_extra["unexpected"] = true + mktemp() do temporary, output + TOML.print(output, with_extra) + flush(output) + @test_throws ArgumentError begin + n5externalloadmanifest(temporary) + end + end + with_missing = TOML.parsefile(N5_EXTERNAL_MANIFEST_PATH) + delete!(first(with_missing["case"]), "rows") + mktemp() do temporary, output + TOML.print(output, with_missing) + flush(output) + @test_throws ArgumentError begin + n5externalloadmanifest(temporary) + end + end +end + +@testset "N5 Julia external fixture read and rewrite" begin + for fixture in N5_EXTERNAL_MANIFEST.fixtures + identity = "$(fixture.producer)/$(fixture.case_id)/$(fixture.page_version)" + @testset "$identity" begin + path = joinpath(N5_EXTERNAL_ROOT, split(fixture.file, '/')...) + table = Parquet.Table(path) + try + expected = N5_EXTERNAL_EXPECTED_ROWS[ + "$(fixture.producer)/$(fixture.case_id)"] + @test table.rows == fixture.rows == length(expected) + @test table.metadata.num_rows == fixture.rows + @test all(==(fixture.page_version), + n5externalpageversions(table)) + actual = n5externalnormalizerows(table) + @test isequal(actual, expected) + rewrites = Dict{Symbol,Vector{UInt8}}() + for pageversion in (:v1, :v2) + output = n5externalrewrite(table, pageversion) + @test output.privatefirst == output.privatesecond + @test output.publicfirst == output.publicsecond + @test output.privatefirst == output.publicfirst + rewrites[pageversion] = output.privatefirst + decoded = N5.n5decodefile(output.privatefirst) + @test decoded.metadata.num_rows == fixture.rows + @test N5.n5schemaexact(decoded.metadata.schema, + table.metadata.schema) + @test all(==(pageversion), decoded.pageversions) + rewritten = Parquet.Table(output.privatefirst) + try + @test rewritten.rows == fixture.rows + @test N5.n5schemaexact(rewritten.metadata.schema, + table.metadata.schema) + @test all(==(pageversion), + n5externalpageversions(rewritten)) + @test isequal(n5externalnormalizerows(rewritten), + expected) + finally + close(rewritten) + end + end + @test rewrites[:v1] != rewrites[:v2] + finally + close(table) + end + end + end +end + +@testset "N5 external optional-key actual-null mutation" begin + fixture = only(item for item in N5_EXTERNAL_MANIFEST.fixtures if + item.producer == "arrow-rs" && + item.case_id == "arrow-rs-optional-key-present" && + item.page_version === :v1) + source = read(joinpath(N5_EXTERNAL_ROOT, split(fixture.file, '/')...)) + decoded = N5.n5decodefile(source) + @test decoded.metadata.num_rows == fixture.rows + keyindex = findfirst(leaf -> leaf.path == + ["entries", "key_value", "key"], decoded.leaves) + @test keyindex !== nothing + if keyindex !== nothing + key = decoded.streams[keyindex] + @test key.max_repetition == 1 + @test key.max_definition == 3 + @test key.repetition == UInt64[0, 0, 0, 0, 1] + @test key.definition == UInt64[0, 1, 3, 3, 3] + @test key.values == Any["a", "b", "c"] + definitions = copy(key.definition) + definitions[3] = UInt64(2) + streams = copy(decoded.streams) + streams[keyindex] = N5.N5LeafStream(copy(key.repetition), + definitions, Any[key.values[2:end]...], key.max_repetition, + key.max_definition) + for pageversion in (:v1, :v2) + mutated = N5.n5emitfile(decoded.metadata.schema, streams, + fixture.rows; pageversion=pageversion) + wire = N5.n5decodefile(mutated) + @test N5.n5schemaexact(wire.metadata.schema, + decoded.metadata.schema) + @test wire.streams[keyindex].definition[3] == UInt64(2) + @test wire.streams[keyindex].values == Any["b", "c"] + @test n5externaltableerror(mutated) isa Parquet.FormatError + end + end +end diff --git a/test/conformance/n5/generate-codec-candidates.jl b/test/conformance/n5/generate-codec-candidates.jl new file mode 100755 index 0000000..d5ab90f --- /dev/null +++ b/test/conformance/n5/generate-codec-candidates.jl @@ -0,0 +1,122 @@ +#!/usr/bin/env julia + +using Parquet +using SHA + +include(joinpath(@__DIR__, "model", "N5ConformanceModel.jl")) + +const N5 = N5ConformanceModel + +function n5usage() + println(stderr, "usage: generate-codec-candidates.jl --output DIR " * + "[--codec-ids ID,ID,...]") + return exit(64) +end + +function n5arguments(arguments::Vector{String}) + values = Dict{String,String}() + index = 1 + while index <= length(arguments) + index == length(arguments) && n5usage() + key = arguments[index] + key in ("--output", "--codec-ids") || n5usage() + haskey(values, key) && n5usage() + values[key] = arguments[index + 1] + index += 2 + end + haskey(values, "--output") || n5usage() + ids = if haskey(values, "--codec-ids") + parsed = parse.(Int, split(values["--codec-ids"], ',')) + length(parsed) == 32 || throw(ArgumentError( + "--codec-ids must contain exactly 32 IDs")) + length(Set(parsed)) == 32 || throw(ArgumentError( + "--codec-ids contains a duplicate")) + sort(parsed) == parsed || throw(ArgumentError( + "--codec-ids must be strictly increasing")) + parsed + else + nothing + end + return abspath(values["--output"]), ids +end + +function n5write(path::String, bytes::Vector{UInt8}) + open(path, "w") do output + write(output, bytes) + return nothing + end + return bytes2hex(SHA.sha256(bytes)) +end + +function n5writecandidates(output::String, cases) + manifest = joinpath(output, "cases.tsv") + open(manifest, "w") do io + println(io, join(("id", "page_version", "rows", "payload_bytes", + "depth", "width", "ast_nodes", "leaves", "level_entries", + "dense_values", "sha256", "file"), '\t')) + for case in cases + length(case.rows) > 0 && case.payloadbytes > 0 || continue + file = "case-$(lpad(case.id, 4, '0')).parquet" + bytes = N5.n5emitfile(case.schema, case.streams, + length(case.rows); pageversion=case.pageversion) + digest = n5write(joinpath(output, file), bytes) + println(io, join((case.id, case.pageversion, length(case.rows), + case.payloadbytes, case.depth, case.width, case.astnodes, + case.leaves, case.levelentries, case.densevalues, digest, + file), '\t')) + end + return nothing + end + println("generated $(count(line -> endswith(line, ".parquet"), + readdir(output))) codec candidate schemas") + return nothing +end + +function n5writecodecs(output::String, cases, ids::Vector{Int}) + byid = Dict(case.id => case for case in cases) + manifest = joinpath(output, "codecs.tsv") + count = 0 + open(manifest, "w") do io + println(io, join(("id", "page_version", "codec", "sha256", + "file"), '\t')) + for id in ids + case = get(byid, id, nothing) + case === nothing && throw(ArgumentError( + "codec candidate ID $id is not accepted")) + length(case.rows) > 0 && case.payloadbytes > 0 || + throw(ArgumentError("codec candidate ID $id has no payload")) + for codec in N5.N5_PROPERTY_CODECS + fixture = N5.n5propertycodecfixture(case, codec) + digest = n5write(joinpath(output, fixture.filename), + fixture.bytes) + digest == fixture.sha256 || throw(ArgumentError( + "codec fixture $(fixture.filename) SHA-256 differs")) + println(io, join((id, case.pageversion, codec, digest, + fixture.filename), '\t')) + count += 1 + end + end + return nothing + end + count == 192 || throw(ArgumentError( + "expected 192 codec fixtures, got $count")) + println("generated $count codec candidate fixtures") + return nothing +end + +function main(arguments::Vector{String}) + output, ids = n5arguments(arguments) + if ispath(output) + isdir(output) || throw(ArgumentError("--output is not a directory")) + isempty(readdir(output)) || throw(ArgumentError( + "--output must be empty")) + else + mkpath(output) + end + cases, _ = N5.n5propertycases() + ids === nothing ? n5writecandidates(output, cases) : + n5writecodecs(output, cases, ids) + return nothing +end + +main(ARGS) diff --git a/test/conformance/n5/generate-julia-fixtures.jl b/test/conformance/n5/generate-julia-fixtures.jl new file mode 100755 index 0000000..13f59b4 --- /dev/null +++ b/test/conformance/n5/generate-julia-fixtures.jl @@ -0,0 +1,361 @@ +#!/usr/bin/env julia + +using Parquet +using SHA +using TOML + +include(joinpath(@__DIR__, "model", "N5ConformanceModel.jl")) + +const N5 = N5ConformanceModel +const N5_PAGE_VERSIONS = (:v1, :v2) + +struct N5FixtureMapping + kind::String + caseid::String + pageversion::Symbol + codec::Symbol + reference::String + target::String + referencesha256::String + targetsha256::String +end + +function n5usage() + println(stderr, "usage: generate-julia-fixtures.jl --repo REPO --output DIR") + return exit(64) +end + +function n5arguments(arguments::Vector{String}) + values = Dict{String,String}() + index = 1 + while index <= length(arguments) + index == length(arguments) && n5usage() + key = arguments[index] + key in ("--repo", "--output") || n5usage() + haskey(values, key) && n5usage() + values[key] = arguments[index + 1] + index += 2 + end + Set(keys(values)) == Set(("--repo", "--output")) || n5usage() + return abspath(values["--repo"]), abspath(values["--output"]) +end + +function n5relativepath(path::String, label::String) + isempty(path) && throw(ArgumentError("$label must not be empty")) + isabspath(path) && throw(ArgumentError("$label must be relative")) + occursin('\\', path) && throw(ArgumentError( + "$label must use forward slashes")) + parts = split(path, '/') + any(part -> isempty(part) || part in (".", ".."), parts) && + throw(ArgumentError("$label is not normalized")) + normpath(path) == path || throw(ArgumentError( + "$label is not normalized")) + return path +end + +function n5sha256(bytes::AbstractVector{UInt8}) + return bytes2hex(SHA.sha256(bytes)) +end + +function n5sha256(path::String) + return open(path, "r") do input + return bytes2hex(SHA.sha256(input)) + end +end + +function n5writebytes(root::String, relative::String, + bytes::AbstractVector{UInt8}, seen::Set{String}) + n5relativepath(relative, "generated fixture path") + relative in seen && throw(ArgumentError( + "duplicate generated fixture path $relative")) + push!(seen, relative) + path = joinpath(root, split(relative, '/')...) + mkpath(dirname(path)) + open(path, "w") do output + write(output, bytes) + return nothing + end + return n5sha256(bytes) +end + +function n5mapping(kind::String, caseid::String, pageversion::Symbol, + codec::Symbol, reference::String, target::String, + referencesha256::String, targetsha256::String) + pageversion in N5_PAGE_VERSIONS || throw(ArgumentError( + "unsupported mapping page version $pageversion")) + n5relativepath(reference, "reference path") + n5relativepath(target, "target path") + return N5FixtureMapping(kind, caseid, pageversion, codec, reference, + target, referencesha256, targetsha256) +end + +function n5rewrite(source::Vector{UInt8}, pageversion::Symbol; + codec::Symbol=:uncompressed) + table = Parquet.Table(source) + try + output = N5.n5productionencodedbytes(table, pageversion; codec=codec) + output.privatefirst == output.privatesecond == output.publicfirst == + output.publicsecond || throw(ArgumentError( + "production output is not byte deterministic")) + return output.publicfirst + finally + close(table) + end +end + +function n5checkmodel(case, source::Vector{UInt8}, target::Vector{UInt8}, + pageversion::Symbol) + sourcedecoded = N5.n5decodefile(source) + targetdecoded = N5.n5decodefile(target) + N5.n5schemaexact(sourcedecoded.metadata.schema, + targetdecoded.metadata.schema) || throw(ArgumentError( + "$(case.name) schema differs after production rewrite")) + sourcedecoded.metadata.num_rows == targetdecoded.metadata.num_rows == + length(case.rows) || throw(ArgumentError( + "$(case.name) row count differs after production rewrite")) + sourcedecoded.streams == targetdecoded.streams || throw(ArgumentError( + "$(case.name) streams differ after production rewrite")) + all(==(pageversion), targetdecoded.pageversions) || throw(ArgumentError( + "$(case.name) production page version differs")) + return nothing +end + +function n5checkproperty(case, fixture) + table = Parquet.Table(fixture.bytes) + try + table.rows == length(case.rows) || throw(ArgumentError( + "property case $(case.id) row count differs")) + N5.n5schemaexact(table.metadata.schema, fixture.schema) || + throw(ArgumentError("property case $(case.id) schema differs")) + isequal(N5.n5normalizetable(case.node, table), case.rows) || + throw(ArgumentError("property case $(case.id) rows differ")) + finally + close(table) + end + return nothing +end + +function n5modelbindings!(output::String, mappings::Vector{N5FixtureMapping}, + seen::Set{String}) + cases = Any[N5.n5bindinggoldens()...] + push!(cases, N5.n5provenancegolden()) + for case in cases + kind = case.name == "schema-provenance" ? "provenance" : "binding" + for pageversion in N5_PAGE_VERSIONS + reference = "reference/model/$(case.name).$(pageversion).parquet" + target = "julia/model/$(case.name).$(pageversion).parquet" + source = N5.n5emitfile(case.schema, case.streams, + length(case.rows); pageversion=pageversion) + rewritten = n5rewrite(source, pageversion) + n5checkmodel(case, source, rewritten, pageversion) + referencesha = n5writebytes(output, reference, source, seen) + targetsha = n5writebytes(output, target, rewritten, seen) + push!(mappings, n5mapping(kind, case.name, pageversion, + :uncompressed, reference, target, referencesha, targetsha)) + end + end + return nothing +end + +function n5properties!(output::String, mappings::Vector{N5FixtureMapping}, + seen::Set{String}) + cases, _ = N5.n5propertycases() + subset = N5.n5propertycodecsubset(cases) + fixtures = N5.n5propertycodecfixtures(cases) + byid = Dict(case.id => case for case in subset) + references = Dict{Int,Tuple{String,String}}() + for case in subset + reference = "reference/property/$(case.name)-$(case.pageversion).parquet" + source = N5.n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=case.pageversion) + references[case.id] = (reference, + n5writebytes(output, reference, source, seen)) + end + for fixture in fixtures + case = get(byid, fixture.caseid, nothing) + case === nothing && throw(ArgumentError( + "unknown property codec case $(fixture.caseid)")) + fixture.pageversion == case.pageversion || throw(ArgumentError( + "property codec page version differs for $(fixture.caseid)")) + fixture.sha256 == n5sha256(fixture.bytes) || throw(ArgumentError( + "property codec SHA-256 differs for $(fixture.filename)")) + target = "julia/property/$(fixture.filename)" + targetsha = n5writebytes(output, target, fixture.bytes, seen) + reference, referencesha = references[fixture.caseid] + n5checkproperty(case, fixture) + push!(mappings, n5mapping("property", string(fixture.caseid), + fixture.pageversion, fixture.codec, reference, target, + referencesha, targetsha)) + end + return nothing +end + +function n5normalizeread(value) + value === missing && return missing + if value isa Parquet.StructValue + return (:struct, Pair{String,Any}[String(pair.first) => + n5normalizeread(pair.second) for pair in value]) + elseif value isa Parquet.MapValue + return (:map, Pair{Any,Any}[n5normalizeread(pair.first) => + n5normalizeread(pair.second) for pair in value]) + elseif value isa Parquet.ListValue + return (:list, Any[n5normalizeread(item) for item in value]) + elseif value isa AbstractVector{UInt8} + return (:bytes, bytes2hex(value)) + elseif value isa AbstractVector + throw(ArgumentError("unexpected materialized vector $(typeof(value))")) + end + return value +end + +function n5normalizetable(table::Parquet.Table) + columns = collect(pairs(table.columns)) + output = Vector{Vector{Pair{String,Any}}}(undef, table.rows) + for row in 1:table.rows + values = Pair{String,Any}[] + sizehint!(values, length(columns)) + for column in columns + push!(values, String(column.first) => + n5normalizeread(column.second[row])) + end + output[row] = values + end + return output +end + +function n5external!(repo::String, output::String, + mappings::Vector{N5FixtureMapping}, seen::Set{String}) + root = joinpath(repo, "test", "conformance", "n5") + manifest = TOML.parsefile(joinpath(root, "manifest.toml")) + rawcases = get(manifest, "case", nothing) + rawcases isa AbstractVector || throw(ArgumentError( + "external manifest case list is absent")) + fixtures = NamedTuple[] + for rawcase in rawcases + producer = String(rawcase["producer"]) + caseid = String(rawcase["case_id"]) + files = rawcase["files"] + files isa AbstractVector || throw(ArgumentError( + "external case $caseid has no files")) + for rawfile in files + pageversion = Symbol(String(rawfile["page_version"])) + sourcepath = String(rawfile["file"]) + n5relativepath(sourcepath, "external fixture path") + push!(fixtures, (; producer, caseid, pageversion, sourcepath)) + end + end + sort!(fixtures; by=fixture -> (fixture.producer, fixture.caseid, + fixture.pageversion)) + for fixture in fixtures + input = read(joinpath(root, split(fixture.sourcepath, '/')...)) + reference = "reference/external/$(fixture.producer)/" * + basename(fixture.sourcepath) + target = "julia/rewrite/$(fixture.producer)/" * + basename(fixture.sourcepath) + rewritten = n5rewrite(input, fixture.pageversion) + sourcetable = Parquet.Table(input) + targettable = Parquet.Table(rewritten) + try + N5.n5schemaexact(sourcetable.metadata.schema, + targettable.metadata.schema) || throw(ArgumentError( + "external $(fixture.caseid) schema differs")) + sourcetable.rows == targettable.rows || throw(ArgumentError( + "external $(fixture.caseid) rows differ")) + isequal(n5normalizetable(sourcetable), + n5normalizetable(targettable)) || throw(ArgumentError( + "external $(fixture.caseid) values differ")) + finally + close(targettable) + close(sourcetable) + end + result = N5.n5decodefile(rewritten) + all(==(fixture.pageversion), result.pageversions) || throw( + ArgumentError("external $(fixture.caseid) page version differs")) + referencesha = n5writebytes(output, reference, input, seen) + targetsha = n5writebytes(output, target, rewritten, seen) + push!(mappings, n5mapping("external", fixture.caseid, + fixture.pageversion, :uncompressed, reference, target, + referencesha, targetsha)) + end + return nothing +end + +function n5writemappingmanifest(output::String, + mappings::Vector{N5FixtureMapping}, seen::Set{String}) + sort!(mappings; by=mapping -> (mapping.target, mapping.reference)) + length(Set(mapping.target for mapping in mappings)) == length(mappings) || + throw(ArgumentError("generated target paths are not unique")) + path = "fixture-manifest.tsv" + path in seen && throw(ArgumentError("duplicate generated manifest path")) + push!(seen, path) + open(joinpath(output, path), "w") do io + println(io, join(("kind", "case_id", "page_version", "codec", + "reference", "target", "reference_sha256", "target_sha256"), + '\t')) + for mapping in mappings + fields = (mapping.kind, mapping.caseid, + String(mapping.pageversion), String(mapping.codec), + mapping.reference, mapping.target, mapping.referencesha256, + mapping.targetsha256) + any(field -> occursin('\t', field) || occursin('\n', field), + fields) && throw(ArgumentError( + "generated manifest field contains a delimiter")) + println(io, join(fields, '\t')) + end + return nothing + end + return nothing +end + +function n5walkfiles(root::String) + files = String[] + for (directory, _, names) in walkdir(root) + for name in names + relative = replace(relpath(joinpath(directory, name), root), + '\\' => '/') + push!(files, relative) + end + end + sort!(files) + return files +end + +function n5writefilemanifest(output::String, seen::Set{String}) + actual = n5walkfiles(output) + sort!(collect(seen)) == actual || throw(ArgumentError( + "generated file set differs before the file manifest")) + open(joinpath(output, "files.sha256"), "w") do io + for relative in actual + digest = n5sha256(joinpath(output, split(relative, '/')...)) + println(io, "$digest $relative") + end + return nothing + end + return nothing +end + +function main(arguments::Vector{String}) + repo, output = n5arguments(arguments) + isfile(joinpath(repo, "Project.toml")) || throw(ArgumentError( + "--repo does not name the Parquet repository")) + if ispath(output) + isdir(output) || throw(ArgumentError("--output is not a directory")) + isempty(readdir(output)) || throw(ArgumentError( + "--output must be empty")) + else + mkpath(output) + end + mappings = N5FixtureMapping[] + seen = Set{String}() + n5modelbindings!(output, mappings, seen) + n5properties!(output, mappings, seen) + n5external!(repo, output, mappings, seen) + length(mappings) == 256 || throw(ArgumentError( + "expected 256 Julia fixture mappings, got $(length(mappings))")) + n5writemappingmanifest(output, mappings, seen) + n5writefilemanifest(output, seen) + println("generated $(length(mappings)) N5 Julia fixture mappings") + return nothing +end + +main(ARGS) diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v1.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v1.parquet new file mode 100644 index 0000000..7e55c51 Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v1.parquet differ diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v2.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v2.parquet new file mode 100644 index 0000000..d51755e Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-duplicate-keys_v2.parquet differ diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet new file mode 100644 index 0000000..96470aa Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet differ diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet new file mode 100644 index 0000000..5fb8dc2 Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet differ diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v1.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v1.parquet new file mode 100644 index 0000000..3c23603 Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v1.parquet differ diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v2.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v2.parquet new file mode 100644 index 0000000..78e4ea3 Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-list-rule3_v2.parquet differ diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v1.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v1.parquet new file mode 100644 index 0000000..86ca36b Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v1.parquet differ diff --git a/test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v2.parquet b/test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v2.parquet new file mode 100644 index 0000000..6e77cd4 Binary files /dev/null and b/test/conformance/n5/golden/arrow-rs/arrow-rs-optional-key-present_v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_direct_map.v1.parquet b/test/conformance/n5/golden/parquet-java/list_direct_map.v1.parquet new file mode 100644 index 0000000..89ad90b Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_direct_map.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_direct_map.v2.parquet b/test/conformance/n5/golden/parquet-java/list_direct_map.v2.parquet new file mode 100644 index 0000000..aa948d6 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_direct_map.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v1.parquet b/test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v1.parquet new file mode 100644 index 0000000..6175972 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v2.parquet b/test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v2.parquet new file mode 100644 index 0000000..74b037f Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_direct_map_utf8.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule1_primitive.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule1_primitive.v1.parquet new file mode 100644 index 0000000..147de4e Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule1_primitive.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule1_primitive.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule1_primitive.v2.parquet new file mode 100644 index 0000000..1892c07 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule1_primitive.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule2_struct.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule2_struct.v1.parquet new file mode 100644 index 0000000..0fd733b Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule2_struct.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule2_struct.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule2_struct.v2.parquet new file mode 100644 index 0000000..660881e Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule2_struct.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule3_nested.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule3_nested.v1.parquet new file mode 100644 index 0000000..d3e759c Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule3_nested.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule3_nested.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule3_nested.v2.parquet new file mode 100644 index 0000000..f9d64ec Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule3_nested.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet new file mode 100644 index 0000000..b896c45 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet new file mode 100644 index 0000000..17e36fc Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule4_array.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule4_array.v1.parquet new file mode 100644 index 0000000..dae8409 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule4_array.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule4_array.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule4_array.v2.parquet new file mode 100644 index 0000000..9ea5d76 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule4_array.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule4_tuple.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule4_tuple.v1.parquet new file mode 100644 index 0000000..4f0b5aa Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule4_tuple.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule4_tuple.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule4_tuple.v2.parquet new file mode 100644 index 0000000..c7a5ea7 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule4_tuple.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v1.parquet new file mode 100644 index 0000000..b5c3dc2 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v2.parquet new file mode 100644 index 0000000..07936fd Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule5_optional_extended.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v1.parquet new file mode 100644 index 0000000..5d0efa6 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v2.parquet new file mode 100644 index 0000000..69bff6e Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule5_optional_paired.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule5_required.v1.parquet b/test/conformance/n5/golden/parquet-java/list_rule5_required.v1.parquet new file mode 100644 index 0000000..454666e Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule5_required.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/list_rule5_required.v2.parquet b/test/conformance/n5/golden/parquet-java/list_rule5_required.v2.parquet new file mode 100644 index 0000000..ee2246c Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/list_rule5_required.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_arbitrary_names.v1.parquet b/test/conformance/n5/golden/parquet-java/map_arbitrary_names.v1.parquet new file mode 100644 index 0000000..8d8065c Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_arbitrary_names.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_arbitrary_names.v2.parquet b/test/conformance/n5/golden/parquet-java/map_arbitrary_names.v2.parquet new file mode 100644 index 0000000..c717ab0 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_arbitrary_names.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_key_only.v1.parquet b/test/conformance/n5/golden/parquet-java/map_key_only.v1.parquet new file mode 100644 index 0000000..9d615fd Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_key_only.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_key_only.v2.parquet b/test/conformance/n5/golden/parquet-java/map_key_only.v2.parquet new file mode 100644 index 0000000..82f4240 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_key_only.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_standalone_mkv.v1.parquet b/test/conformance/n5/golden/parquet-java/map_standalone_mkv.v1.parquet new file mode 100644 index 0000000..5b76d0e Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_standalone_mkv.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_standalone_mkv.v2.parquet b/test/conformance/n5/golden/parquet-java/map_standalone_mkv.v2.parquet new file mode 100644 index 0000000..0fb59a6 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_standalone_mkv.v2.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_standard.v1.parquet b/test/conformance/n5/golden/parquet-java/map_standard.v1.parquet new file mode 100644 index 0000000..fb64921 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_standard.v1.parquet differ diff --git a/test/conformance/n5/golden/parquet-java/map_standard.v2.parquet b/test/conformance/n5/golden/parquet-java/map_standard.v2.parquet new file mode 100644 index 0000000..8d38ab3 Binary files /dev/null and b/test/conformance/n5/golden/parquet-java/map_standard.v2.parquet differ diff --git a/test/conformance/n5/hardening/runtests.jl b/test/conformance/n5/hardening/runtests.jl new file mode 100644 index 0000000..1981b4b --- /dev/null +++ b/test/conformance/n5/hardening/runtests.jl @@ -0,0 +1,4 @@ +using Parquet +using Test + +include("source_mutation.jl") diff --git a/test/conformance/n5/hardening/source_mutation.jl b/test/conformance/n5/hardening/source_mutation.jl new file mode 100644 index 0000000..0257f1c --- /dev/null +++ b/test/conformance/n5/hardening/source_mutation.jl @@ -0,0 +1,3655 @@ +import Tables +import Dates + +mutable struct N5HActionVector{T,F} <: AbstractVector{T} + values::Vector{T} + calls::Int + trigger::Int + action::F +end + +function N5HActionVector(values::Vector{T}, trigger::Int, action::F) where {T,F} + return N5HActionVector{T,F}(values, 0, trigger, action) +end + +function N5HActionVector(action::F, values::Vector{T}, trigger::Int) where {T,F} + return N5HActionVector(values, trigger, action) +end + +function Base.IndexStyle(::Type{<:N5HActionVector}) + return IndexLinear() +end + +function Base.size(values::N5HActionVector) + return size(values.values) +end + +function Base.getindex(values::N5HActionVector, index::Int) + values.calls += 1 + values.calls == values.trigger && values.action() + return values.values[index] +end + +mutable struct N5HSequenceVector{T} <: AbstractVector{T} + values::Vector{T} + calls::Int +end + +function Base.IndexStyle(::Type{<:N5HSequenceVector}) + return IndexLinear() +end + +function Base.size(::N5HSequenceVector) + return (1,) +end + +function Base.getindex(values::N5HSequenceVector, ::Int) + values.calls += 1 + return values.values[min(values.calls, length(values.values))] +end + +mutable struct N5HAxisVector{T} <: AbstractVector{T} + values::Vector{T} + shifted::Bool +end + +mutable struct N5HBackingAxisVector{T} <: AbstractVector{T} + values::Vector{T} + calls::Int + trigger::Int + shifted::Bool +end + +function Base.IndexStyle(::Type{<:N5HAxisVector}) + return IndexCartesian() +end + +function Base.size(values::N5HAxisVector) + return size(values.values) +end + +function Base.axes(values::N5HAxisVector) + count = length(values.values) + return values.shifted ? (0:(count - 1),) : (Base.OneTo(count),) +end + +function Base.getindex(values::N5HAxisVector, index::Int) + values.shifted = true + return values.values[index] +end + +function Base.IndexStyle(::Type{<:N5HBackingAxisVector}) + return IndexCartesian() +end + +function Base.size(values::N5HBackingAxisVector) + return size(values.values) +end + +function Base.axes(values::N5HBackingAxisVector) + count = length(values.values) + return values.shifted ? (0:(count - 1),) : (Base.OneTo(count),) +end + +function Base.getindex(values::N5HBackingAxisVector, index::Int) + values.calls += 1 + values.calls == values.trigger && (values.shifted = true) + return values.values[index] +end + +mutable struct N5HAlternatingDict <: AbstractDict{Int32,Int32} + passes::Int +end + +struct N5HCountDict <: AbstractDict{Int32,Int32} + count::Int +end + +struct N5HLyingCycleDict <: AbstractDict{Int32,Int32} end + +mutable struct N5HFinalMutKeyDict{K,V} <: AbstractDict{K,V} + item::Pair{K,V} + mutated::Bool + attacks::Int +end + +mutable struct N5HFinalMutMap{K,V} <: AbstractDict{K,V} + item::Pair{K,V} + mutated::Bool + attacks::Int +end + +mutable struct N5HNothingStateDict{K,V} <: AbstractDict{K,V} + item::Pair{K,V} + calls::Int + length_calls::Int +end + +mutable struct N5HRestoringChild <: AbstractVector{Int32} + owner::Base.RefValue{Any} + armed::Bool + restores::Int +end + +mutable struct N5HCountingVector <: AbstractVector{Int32} + calls::Int +end + +mutable struct N5HParentAfterChildDict <: AbstractDict{Int32,Any} + parent::Parquet.ListVector + attacks::Int +end + +mutable struct N5HChangingSourceDict{V,E} <: AbstractDict{Int32,V} + old::V + new::V + starts::Int + terminals::Int + error::E +end + +struct N5HSecondTouchDict{V} <: AbstractDict{Int32,Any} + second::V +end + +struct N5HRestoreVector{T,D} <: AbstractVector{T} + value::T + target::D + key::Bool +end + +mutable struct N5HLengthMutationVector{T,F} <: AbstractVector{T} + values::Vector{T} + action::F + armed::Bool + calls::Int +end + +mutable struct N5HSequencedByteKey <: AbstractVector{UInt8} + passes::Int + attack::Int +end + +struct N5HBadByteKey <: AbstractVector{UInt8} end + +struct N5HThrowByteKey{E} <: AbstractVector{UInt8} + error::E +end + +mutable struct N5HAxisShiftByteKey <: AbstractVector{UInt8} + shifted::Bool +end + +mutable struct N5HShiftBadByteKey <: AbstractVector{UInt8} + shifted::Bool +end + +mutable struct N5HSequencedShiftBadByteKey <: AbstractVector{UInt8} + calls::Int + attack::Int + shifted::Bool +end + +mutable struct N5HSequencedLengthByteKey <: AbstractVector{UInt8} + calls::Int +end + +struct N5HThrowLengthKey{E} <: AbstractVector{Int32} + error::E +end + +mutable struct N5HChain <: AbstractVector{Any} + child::Any +end + +mutable struct N5HLyingInt32Vector <: AbstractVector{Int32} + child::Any + calls::Int +end + +mutable struct N5HMutableSentinel <: Exception + value::Int +end + +struct N5HThrowAnyVector{E} <: AbstractVector{Any} + error::E +end + +mutable struct N5HHiddenNestedChild <: AbstractVector{Int32} + values::Vector{Int32} + calls::Int + owner::Base.RefValue{Any} +end + +struct N5HThrowMetricChild{E} <: AbstractVector{Int32} + error::E +end + +struct N5HFalseBoundsChild <: AbstractVector{Int32} end + +struct N5HThrowBoundsChild{E} <: AbstractVector{Int32} + error::E +end + +mutable struct N5HShiftOuter{T} <: AbstractVector{T} + value::T + shifted::Bool +end + +mutable struct N5HSequencedString <: AbstractString + calls::Int +end + +mutable struct N5HSequencedCodeunitVector <: AbstractVector{UInt8} + reads::Int +end + +struct N5HSameSourceString <: AbstractString + bytes::N5HSequencedCodeunitVector +end + +struct N5HShiftBadString <: AbstractString + bytes::N5HSequencedShiftBadByteKey +end + +mutable struct N5HNCodeunitAxisBytes <: AbstractVector{UInt8} + shifted::Bool +end + +mutable struct N5HNCodeunitAxisString <: AbstractString + bytes::N5HNCodeunitAxisBytes + calls::Int + attack::Int +end + +function Base.length(values::N5HCountDict) + return values.count +end + +function Base.getindex(::N5HCountDict, key::Int32) + return Int32(10) * key +end + +function Base.iterate(values::N5HCountDict, state::Int=1) + state > values.count && return nothing + key = Int32(state) + return key => values[key], state + 1 +end + +function Base.length(::N5HLyingCycleDict) + return 1 +end + +function Base.getindex(::N5HLyingCycleDict, ::Int32) + return Int32(1) +end + +function Base.iterate(values::N5HLyingCycleDict, state::Int=1) + state == 1 || return nothing + return values => Int32(1), 2 +end + +function N5HFinalMutKeyDict(item::Pair{K,V}) where {K,V} + return N5HFinalMutKeyDict{K,V}(item, false, 0) +end + +function N5HFinalMutMap(item::Pair{K,V}) where {K,V} + return N5HFinalMutMap{K,V}(item, false, 0) +end + +function Base.length(values::N5HFinalMutKeyDict) + if values.mutated + values.item.first.offsets[2] = 1 + values.mutated = false + end + return 1 +end + +function Base.length(values::N5HFinalMutMap) + if values.mutated + values.item.second.offsets[2] = 1 + values.mutated = false + end + return 1 +end + +function Base.getindex(values::N5HFinalMutKeyDict{K,V}, key::K) where {K,V} + isequal(key, values.item.first) || throw(KeyError(key)) + return values.item.second +end + +function Base.getindex(values::N5HFinalMutMap{K,V}, key::K) where {K,V} + isequal(key, values.item.first) || throw(KeyError(key)) + return values.item.second +end + +function Base.iterate(values::N5HFinalMutKeyDict, state::Int=1) + state == 1 && return values.item, 2 + values.item.first.offsets[2] = 0 + values.mutated = true + values.attacks += 1 + return nothing +end + +function Base.iterate(values::N5HFinalMutMap, state::Int=1) + state == 1 && return values.item, 2 + values.item.second.offsets[2] = 0 + values.mutated = true + values.attacks += 1 + return nothing +end + +function Base.length(values::N5HNothingStateDict) + values.length_calls += 1 + throw(AssertionError("dictionary length must not be called")) +end + +function Base.getindex(values::N5HNothingStateDict{K,V}, key::K) where {K,V} + isequal(key, values.item.first) || throw(KeyError(key)) + return values.item.second +end + +function Base.iterate(values::N5HNothingStateDict) + values.calls += 1 + return values.item, nothing +end + +function Base.iterate(values::N5HNothingStateDict, ::Nothing) + values.calls += 1 + return nothing +end + +function Base.IndexStyle(::Type{N5HRestoringChild}) + return IndexLinear() +end + +function Base.size(::N5HRestoringChild) + return (1,) +end + +function Base.axes(::N5HRestoringChild) + return (Base.OneTo(1),) +end + +function Base.length(values::N5HRestoringChild) + owner = values.owner[] + if values.armed && owner !== nothing && owner.offsets[2] == 0 + owner.offsets[2] = 1 + values.restores += 1 + end + return 1 +end + +function Base.getindex(::N5HRestoringChild, index::Int) + index == 1 || throw(BoundsError(index)) + return Int32(11) +end + +function Base.IndexStyle(::Type{N5HCountingVector}) + return IndexLinear() +end + +function Base.size(values::N5HCountingVector) + values.calls += 1 + return (1,) +end + +function Base.getindex(::N5HCountingVector, index::Int) + index == 1 || throw(BoundsError(index)) + return Int32(11) +end + +function Base.length(::N5HParentAfterChildDict) + throw(AssertionError("dictionary length must not be called")) +end + +function Base.getindex(values::N5HParentAfterChildDict, key::Int32) + key == 1 && return Parquet.ListValue(values.parent.values, 1, 1) + key == 2 && return values.parent + throw(KeyError(key)) +end + +function Base.iterate(values::N5HParentAfterChildDict, state::Int=1) + state == 1 && return Int32(1) => values[Int32(1)], 2 + state == 2 && return Int32(2) => values[Int32(2)], 3 + values.parent.offsets[2] = 0 + values.attacks += 1 + return nothing +end + +function Base.length(::N5HChangingSourceDict) + throw(AssertionError("dictionary length must not be called")) +end + +function Base.getindex(values::N5HChangingSourceDict, key::Int32) + key == 1 || throw(KeyError(key)) + return isone(values.starts) ? values.old : values.new +end + +function Base.iterate(values::N5HChangingSourceDict) + values.starts += 1 + source = isone(values.starts) ? values.old : values.new + return Int32(1) => source, 2 +end + +function Base.iterate(values::N5HChangingSourceDict, state::Int) + state == 2 || throw(ArgumentError("invalid dictionary iterator state")) + values.terminals += 1 + values.starts >= 2 && throw(values.error) + return nothing +end + +function Base.length(::N5HSecondTouchDict) + throw(AssertionError("dictionary length must not be called")) +end + +function Base.getindex(values::N5HSecondTouchDict, key::Int32) + key == 1 && return Int32(1) + key == 2 && return values.second + throw(KeyError(key)) +end + +function Base.iterate(values::N5HSecondTouchDict, state::Int=1) + state == 1 && return Int32(1) => values[Int32(1)], 2 + state == 2 && return Int32(2) => values[Int32(2)], 3 + return nothing +end + +function Base.IndexStyle(::Type{<:N5HRestoreVector}) + return IndexLinear() +end + +function Base.size(::N5HRestoreVector) + return (1,) +end + +function Base.axes(::N5HRestoreVector) + return (Base.OneTo(1),) +end + +function Base.length(values::N5HRestoreVector) + target = values.target + if target.mutated + nested = values.key ? target.item.first : target.item.second + nested.offsets[2] = 1 + target.mutated = false + end + return 1 +end + +function Base.getindex(values::N5HRestoreVector, index::Int) + index == 1 || throw(BoundsError(values, index)) + return values.value +end + +function Base.IndexStyle(::Type{<:N5HLengthMutationVector}) + return IndexLinear() +end + +function Base.size(values::N5HLengthMutationVector) + return size(values.values) +end + +function Base.axes(values::N5HLengthMutationVector) + return axes(values.values) +end + +function Base.length(values::N5HLengthMutationVector) + if values.armed + values.calls += 1 + values.armed = false + values.action() + end + return length(values.values) +end + +function Base.getindex(values::N5HLengthMutationVector, index::Int) + return values.values[index] +end + +function Base.IndexStyle(::Type{N5HSequencedByteKey}) + return IndexLinear() +end + +function Base.size(::N5HSequencedByteKey) + return (1,) +end + +function Base.iterate(key::N5HSequencedByteKey, state::Int=1) + state == 1 || return nothing + key.passes += 1 + byte = key.passes == key.attack ? UInt8(0x62) : UInt8(0x61) + return byte, 2 +end + +function Base.getindex(key::N5HSequencedByteKey, ::Int) + item = iterate(key) + item === nothing && throw(BoundsError(key, 1)) + return first(item) +end + +function Base.IndexStyle(::Type{N5HBadByteKey}) + return IndexLinear() +end + +function Base.size(::N5HBadByteKey) + return (1,) +end + +function Base.getindex(::N5HBadByteKey, ::Int) + return 300 +end + +function Base.IndexStyle(::Type{<:N5HThrowByteKey}) + return IndexLinear() +end + +function Base.size(::N5HThrowByteKey) + return (1,) +end + +function Base.getindex(key::N5HThrowByteKey, ::Int) + throw(key.error) +end + +function Base.IndexStyle(::Type{N5HAxisShiftByteKey}) + return IndexCartesian() +end + +function Base.size(::N5HAxisShiftByteKey) + return (1,) +end + +function Base.axes(key::N5HAxisShiftByteKey) + return key.shifted ? (0:0,) : (Base.OneTo(1),) +end + +function Base.getindex(key::N5HAxisShiftByteKey, ::Int) + key.shifted = true + return UInt8(0x61) +end + +function Base.IndexStyle(::Type{N5HShiftBadByteKey}) + return IndexCartesian() +end + +function Base.size(::N5HShiftBadByteKey) + return (1,) +end + +function Base.axes(key::N5HShiftBadByteKey) + return key.shifted ? (0:0,) : (Base.OneTo(1),) +end + +function Base.getindex(key::N5HShiftBadByteKey, ::Int) + key.shifted = true + return 300 +end + +function Base.IndexStyle(::Type{N5HSequencedShiftBadByteKey}) + return IndexCartesian() +end + +function Base.size(::N5HSequencedShiftBadByteKey) + return (1,) +end + +function Base.axes(key::N5HSequencedShiftBadByteKey) + return key.shifted ? (0:0,) : (Base.OneTo(1),) +end + +function Base.getindex(key::N5HSequencedShiftBadByteKey, ::Int) + key.calls += 1 + if key.calls == key.attack + key.shifted = true + return 300 + end + return UInt8(0x61) +end + +function Base.IndexStyle(::Type{N5HSequencedLengthByteKey}) + return IndexLinear() +end + +function Base.size(::N5HSequencedLengthByteKey) + return (2,) +end + +function Base.length(key::N5HSequencedLengthByteKey) + key.calls += 1 + return key.calls == 1 ? 1 : 2 +end + +function Base.getindex(::N5HSequencedLengthByteKey, ::Int) + return UInt8(0x61) +end + +function Base.IndexStyle(::Type{<:N5HThrowLengthKey}) + return IndexLinear() +end + +function Base.size(::N5HThrowLengthKey) + return (1,) +end + +function Base.length(key::N5HThrowLengthKey) + throw(key.error) +end + +function Base.getindex(::N5HThrowLengthKey, ::Int) + return Int32(1) +end + +function Base.IndexStyle(::Type{N5HChain}) + return IndexLinear() +end + +function Base.size(::N5HChain) + return (1,) +end + +function Base.getindex(value::N5HChain, index::Int) + @boundscheck checkbounds(value, index) + return value.child +end + +function Base.IndexStyle(::Type{N5HLyingInt32Vector}) + return IndexLinear() +end + +function Base.size(::N5HLyingInt32Vector) + return (1,) +end + +function Base.getindex(value::N5HLyingInt32Vector, index::Int) + @boundscheck checkbounds(value, index) + value.calls += 1 + return value.child +end + +function Base.IndexStyle(::Type{<:N5HThrowAnyVector}) + return IndexLinear() +end + +function Base.size(::N5HThrowAnyVector) + return (1,) +end + +function Base.getindex(value::N5HThrowAnyVector, ::Int) + throw(value.error) +end + +function Base.IndexStyle(::Type{N5HHiddenNestedChild}) + return IndexLinear() +end + +function Base.size(value::N5HHiddenNestedChild) + return size(value.values) +end + +function Base.getindex(value::N5HHiddenNestedChild, index::Int) + owner = value.owner[] + if owner !== nothing + value.calls += 1 + phase = mod1(value.calls, 3) + phase == 1 && (owner.offsets[2] = 0) + phase == 3 && (owner.offsets[2] = 1) + end + return value.values[index] +end + +function Base.IndexStyle(::Type{<:N5HThrowMetricChild}) + return IndexLinear() +end + +function Base.size(value::N5HThrowMetricChild) + throw(value.error) +end + +function Base.getindex(::N5HThrowMetricChild, ::Int) + return Int32(0) +end + +function Base.IndexStyle(::Type{N5HFalseBoundsChild}) + return IndexLinear() +end + +function Base.size(::N5HFalseBoundsChild) + return (1,) +end + +function Base.getindex(value::N5HFalseBoundsChild, index::Int) + throw(BoundsError(value, index)) +end + +function Base.checkbounds(::Type{Bool}, ::N5HFalseBoundsChild, ::Int) + return false +end + +function Base.IndexStyle(::Type{<:N5HThrowBoundsChild}) + return IndexLinear() +end + +function Base.size(::N5HThrowBoundsChild) + return (1,) +end + +function Base.getindex(value::N5HThrowBoundsChild, index::Int) + throw(BoundsError(value, index)) +end + +function Base.checkbounds(::Type{Bool}, value::N5HThrowBoundsChild, ::Int) + throw(value.error) +end + +function Base.IndexStyle(::Type{<:N5HShiftOuter}) + return IndexCartesian() +end + +function Base.size(::N5HShiftOuter) + return (1,) +end + +function Base.axes(value::N5HShiftOuter) + return value.shifted ? (0:0,) : (Base.OneTo(1),) +end + +function Base.getindex(value::N5HShiftOuter, ::Int) + value.shifted = true + return value.value +end + +function Base.codeunit(::Type{N5HSequencedString}) + return UInt8 +end + +function Base.ncodeunits(::N5HSequencedString) + return 1 +end + +function Base.codeunit(::N5HSequencedString, index::Integer) + index == 1 || throw(BoundsError(index)) + return UInt8(0x61) +end + +function Base.codeunits(value::N5HSequencedString) + value.calls += 1 + return UInt8[value.calls == 1 ? 0x61 : 0x62] +end + +function Base.IndexStyle(::Type{N5HSequencedCodeunitVector}) + return IndexLinear() +end + +function Base.size(::N5HSequencedCodeunitVector) + return (1,) +end + +function Base.getindex(value::N5HSequencedCodeunitVector, ::Int) + value.reads += 1 + return UInt8(0xff) +end + +function Base.codeunit(::Type{N5HSameSourceString}) + return UInt8 +end + +function Base.ncodeunits(::N5HSameSourceString) + return 1 +end + +function Base.codeunit(value::N5HSameSourceString, index::Integer) + return value.bytes[index] +end + +function Base.codeunits(value::N5HSameSourceString) + return value.bytes +end + +function Base.codeunit(::Type{N5HShiftBadString}) + return UInt8 +end + +function Base.ncodeunits(::N5HShiftBadString) + return 1 +end + +function Base.codeunit(value::N5HShiftBadString, index::Integer) + return value.bytes[Int(index)] +end + +function Base.codeunits(value::N5HShiftBadString) + return value.bytes +end + +function Base.IndexStyle(::Type{N5HNCodeunitAxisBytes}) + return IndexCartesian() +end + +function Base.size(::N5HNCodeunitAxisBytes) + return (1,) +end + +function Base.axes(value::N5HNCodeunitAxisBytes) + return value.shifted ? (0:0,) : (Base.OneTo(1),) +end + +function Base.getindex(::N5HNCodeunitAxisBytes, ::Int) + return UInt8(0x61) +end + +function Base.codeunit(::Type{N5HNCodeunitAxisString}) + return UInt8 +end + +function Base.ncodeunits(value::N5HNCodeunitAxisString) + value.calls += 1 + value.calls == value.attack && (value.bytes.shifted = true) + return 1 +end + +function Base.codeunit(value::N5HNCodeunitAxisString, index::Integer) + return value.bytes[Int(index)] +end + +function Base.codeunits(value::N5HNCodeunitAxisString) + return value.bytes +end + +function Base.length(::N5HAlternatingDict) + return 2 +end + +function Base.getindex(::N5HAlternatingDict, key::Int32) + key == 1 && return Int32(10) + key == 2 && return Int32(20) + throw(KeyError(key)) +end + +function Base.iterate(values::N5HAlternatingDict) + values.passes += 1 + reversed = iseven(values.passes) + key = reversed ? Int32(2) : Int32(1) + return key => values[key], (reversed, 2) +end + +function Base.iterate(values::N5HAlternatingDict, + state::Tuple{Bool,Int}) + reversed, position = state + position > 2 && return nothing + key = reversed ? Int32(1) : Int32(2) + return key => values[key], (reversed, position + 1) +end + +mutable struct N5HLiveTable + names::Vector{Symbol} + values::Vector{AbstractVector} + replacement::AbstractVector + calls::Int + mutateat::Int +end + +function Tables.istable(::Type{N5HLiveTable}) + return true +end + +function Tables.columnaccess(::Type{N5HLiveTable}) + return true +end + +function Tables.columns(table::N5HLiveTable) + return table +end + +function Tables.columnnames(table::N5HLiveTable) + table.calls += 1 + table.calls == table.mutateat && (table.values[1] = table.replacement) + return table.names +end + +function Tables.getcolumn(table::N5HLiveTable, name::Symbol) + index = findfirst(isequal(name), table.names) + index === nothing && throw(KeyError(name)) + return table.values[index] +end + +mutable struct N5HNameOrderTable + names::Vector{Symbol} + values::Vector{AbstractVector} + calls::Int +end + +function Tables.istable(::Type{N5HNameOrderTable}) + return true +end + +function Tables.columnaccess(::Type{N5HNameOrderTable}) + return true +end + +function Tables.columns(table::N5HNameOrderTable) + return table +end + +function Tables.columnnames(table::N5HNameOrderTable) + table.calls += 1 + table.calls == 2 && reverse!(table.names) + return table.names +end + +function Tables.getcolumn(table::N5HNameOrderTable, name::Symbol) + index = findfirst(isequal(name), table.names) + index === nothing && throw(KeyError(name)) + return table.values[index] +end + +mutable struct N5HRenameTable + name::Symbol + value::Vector{Int32} + calls::Int +end + +function Tables.istable(::Type{N5HRenameTable}) + return true +end + +function Tables.columnaccess(::Type{N5HRenameTable}) + return true +end + +function Tables.columns(table::N5HRenameTable) + return table +end + +function Tables.columnnames(table::N5HRenameTable) + table.calls += 1 + table.calls == 2 && (table.name = :renamed) + return (table.name,) +end + +function Tables.getcolumn(table::N5HRenameTable, ::Symbol) + return table.value +end + +struct N5HExtraNames end + +function Base.length(::N5HExtraNames) + return 1 +end + +function Base.iterate(::N5HExtraNames, state::Int=1) + state == 1 && return :a, 2 + state == 2 && return :b, 3 + return nothing +end + +mutable struct N5HBadNamesTable + calls::Int + a::Vector{Int32} + b::Vector{Int32} +end + +function Tables.istable(::Type{N5HBadNamesTable}) + return true +end + +function Tables.columnaccess(::Type{N5HBadNamesTable}) + return true +end + +function Tables.columns(table::N5HBadNamesTable) + return table +end + +function Tables.columnnames(table::N5HBadNamesTable) + table.calls += 1 + return table.calls == 1 ? (:a,) : N5HExtraNames() +end + +function Tables.getcolumn(table::N5HBadNamesTable, name::Symbol) + name === :a && return table.a + name === :b && return table.b + throw(KeyError(name)) +end + +struct N5HSentinelError <: Exception end + +struct N5HThrowVector{T} <: AbstractVector{T} + error::N5HSentinelError +end + +function Base.IndexStyle(::Type{<:N5HThrowVector}) + return IndexLinear() +end + +function Base.size(::N5HThrowVector) + return (1,) +end + +function Base.getindex(values::N5HThrowVector, ::Int) + throw(values.error) +end + +struct N5HZeroAxisVector{T} <: AbstractVector{T} + values::Vector{T} +end + +struct N5HNonIntLengthVector <: AbstractVector{Int32} end + +struct N5HUIntSizeVector <: AbstractVector{Int32} end + +struct N5HUIntAxisVector <: AbstractVector{Int32} end + +struct N5HHugeVector <: AbstractVector{Int32} end + +struct N5HHugeUIntDict <: AbstractDict{Int32,Int32} end + +struct N5HThrowSizeVector{E} <: AbstractVector{Int32} + error::E +end + +function Base.IndexStyle(::Type{<:N5HZeroAxisVector}) + return IndexCartesian() +end + +function Base.size(values::N5HZeroAxisVector) + return size(values.values) +end + +function Base.axes(values::N5HZeroAxisVector) + return (0:(length(values.values) - 1),) +end + +function Base.getindex(values::N5HZeroAxisVector, index::Int) + return values.values[index + 1] +end + +function Base.IndexStyle(::Type{N5HNonIntLengthVector}) + return IndexLinear() +end + +function Base.size(::N5HNonIntLengthVector) + return (1,) +end + +function Base.length(::N5HNonIntLengthVector) + return Int32(1) +end + +function Base.getindex(::N5HNonIntLengthVector, ::Int) + return Int32(1) +end + +function Base.IndexStyle(::Type{N5HUIntSizeVector}) + return IndexLinear() +end + +function Base.size(::N5HUIntSizeVector) + return (UInt(1),) +end + +function Base.length(::N5HUIntSizeVector) + return UInt(1) +end + +function Base.getindex(::N5HUIntSizeVector, ::Int) + return Int32(1) +end + +function Base.IndexStyle(::Type{N5HUIntAxisVector}) + return IndexCartesian() +end + +function Base.size(::N5HUIntAxisVector) + return (1,) +end + +function Base.axes(::N5HUIntAxisVector) + return (UInt(1):UInt(1),) +end + +function Base.getindex(::N5HUIntAxisVector, ::Int) + return Int32(1) +end + +function Base.IndexStyle(::Type{N5HHugeVector}) + return IndexLinear() +end + +function Base.size(::N5HHugeVector) + return (typemax(Int),) +end + +function Base.getindex(::N5HHugeVector, ::Int) + return Int32(1) +end + +function Base.length(::N5HHugeUIntDict) + return typemax(UInt) +end + +function Base.getindex(::N5HHugeUIntDict, key::Int32) + return key +end + +function Base.iterate(::N5HHugeUIntDict, state::Int=1) + state == 1 || return nothing + return Int32(1) => Int32(1), 2 +end + +function Base.IndexStyle(::Type{<:N5HThrowSizeVector}) + return IndexLinear() +end + +function Base.size(values::N5HThrowSizeVector) + throw(values.error) +end + +function Base.getindex(::N5HThrowSizeVector, ::Int) + return Int32(1) +end + +function n5herror(f) + try + f() + return nothing + catch err + return err + end +end + +function n5hchain(depth::Int, leaf=Int32(1)) + depth >= 0 || throw(ArgumentError("chain depth must be nonnegative")) + value = leaf + for _ in 1:depth + value = N5HChain(value) + end + return value +end + +function n5hcycle(depth::Int) + depth >= 1 || throw(ArgumentError("cycle depth must be positive")) + root = N5HChain(nothing) + current = root + for _ in 2:depth + child = N5HChain(nothing) + current.child = child + current = child + end + current.child = root + return root +end + +function n5hlistphasemutation(trigger::Int) + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1, 2], trigger) do + pop!(owner[].offsets) + return + end + column = Parquet.ListVector(Int32[0, 1, 2], child) + owner[] = column + return (items=column,) +end + +function n5hlistaxismutation(trigger::Int) + child = N5HBackingAxisVector(Int32[1, 2], 0, trigger, false) + column = Parquet.ListVector(Int32[0, 1, 2], child) + return (items=column,) +end + +function n5hstructphasemutation(trigger::Int) + owner = Ref{Any}(nothing) + firstchild = N5HActionVector(Int32[1], trigger) do + pop!(owner[].children) + return + end + column = Parquet.StructVector(["x", "y"], + [firstchild, Int32[2]]) + owner[] = column + return (s=column,) +end + +function n5hstructchildsubstitution(trigger::Int) + owner = Ref{Any}(nothing) + firstchild = N5HActionVector(Int32[1], trigger) do + owner[].children[2] = Int32[] + return + end + column = Parquet.StructVector(["x", "y"], + [firstchild, Int32[2]]) + owner[] = column + return (s=column,) +end + +function n5hstructchildidentity(trigger::Int) + owner = Ref{Any}(nothing) + replacement = N5HThrowVector{Int32}(N5HSentinelError()) + firstchild = N5HActionVector(Int32[1], trigger) do + owner[].children[2] = replacement + return + end + column = Parquet.StructVector(["x", "y"], + [firstchild, Int32[2]]) + owner[] = column + return (s=column,) +end + +function n5hstructchildshrink(trigger::Int) + later = Int32[2] + firstchild = N5HActionVector(Int32[1], trigger) do + empty!(later) + return + end + column = Parquet.StructVector(["x", "y"], [firstchild, later]) + return (s=column,) +end + +function n5hnestedlistmutation(trigger::Int) + nested = Parquet.ListVector(Int32[0, 1], Int32[2]) + firstchild = N5HActionVector(Int32[1], trigger) do + nested.offsets[2] = 0 + return + end + column = Parquet.StructVector(["x", "items"], [firstchild, nested]) + return (s=column,), firstchild +end + +function n5hhiddenlistattack() + owner = Ref{Any}(nothing) + child = N5HHiddenNestedChild(Int32[10, 20], 0, owner) + inner = Parquet.ListVector(Int32[0, 1, 2], child) + owner[] = inner + outer = Parquet.ListValue(inner, 1, 2) + return (x=[outer],), child, inner +end + +function n5hhiddenmapattack() + owner = Ref{Any}(nothing) + keys = N5HHiddenNestedChild(Int32[1, 2], 0, owner) + inner = Parquet.MapVector(Int32[0, 1, 2], keys, Int32[10, 20]) + owner[] = inner + outer = Parquet.ListValue(inner, 1, 2) + return (x=[outer],), keys, inner +end + +function n5hhiddendirectlistattack() + owner = Ref{Any}(nothing) + child = N5HHiddenNestedChild(Int32[10, 20], 0, owner) + inner = Parquet.ListVector(Int32[0, 1, 2], child) + owner[] = inner + return (x=[inner],), child, inner +end + +function n5hhiddendirectmapattack() + owner = Ref{Any}(nothing) + keys = N5HHiddenNestedChild(Int32[1, 2], 0, owner) + inner = Parquet.MapVector(Int32[0, 1, 2], keys, Int32[10, 20]) + owner[] = inner + return (x=[inner],), keys, inner +end + +function n5hhiddendirectlistkeyattack() + owner = Ref{Any}(nothing) + child = N5HHiddenNestedChild(Int32[10, 20], 0, owner) + inner = Parquet.ListVector(Int32[0, 1, 2], child) + owner[] = inner + column = Parquet.MapVector(Int32[0, 1], [inner], Int32[7]) + return (m=column,), child, inner +end + +function n5hhiddendirectmapkeyattack() + owner = Ref{Any}(nothing) + keys = N5HHiddenNestedChild(Int32[1, 2], 0, owner) + inner = Parquet.MapVector(Int32[0, 1, 2], keys, Int32[10, 20]) + owner[] = inner + column = Parquet.MapVector(Int32[0, 1], [inner], Int32[7]) + return (m=column,), keys, inner +end + +function n5hmapphasemutation(trigger::Int) + owner = Ref{Any}(nothing) + keys = N5HActionVector(Int32[1], typemax(Int)) do + pop!(something(owner[].values)) + return + end + column = Parquet.MapVector(Int32[0, 1], keys, Int32[10]) + owner[] = column + keys.calls = 0 + keys.trigger = trigger + return (m=column,) +end + +function n5hmapaxismutation(trigger::Int) + values = N5HBackingAxisVector(Int32[10], 0, typemax(Int), false) + keys = N5HActionVector(Int32[1], typemax(Int)) do + values.shifted = true + return + end + column = Parquet.MapVector(Int32[0, 1], keys, values) + keys.calls = 0 + keys.trigger = trigger + return (m=column,) +end + +function n5hstructkeymutation(trigger::Int) + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1], trigger) do + pop!(owner[].children) + return + end + keys = Parquet.StructVector(["x", "y"], [child, Int32[2]]) + owner[] = keys + maps = Parquet.MapVector(Int32[0, 1], keys, Int32[10]) + return (m=maps,) +end + +function n5hstructkeysubstitution(trigger::Int) + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1], trigger) do + owner[].children[2] = Int32[] + return + end + keys = Parquet.StructVector(["x", "y"], [child, Int32[2]]) + owner[] = keys + maps = Parquet.MapVector(Int32[0, 1], keys, Int32[10]) + return (m=maps,) +end + +function n5hstructkeyidentity(trigger::Int) + owner = Ref{Any}(nothing) + replacement = N5HThrowVector{Int32}(N5HSentinelError()) + child = N5HActionVector(Int32[1], trigger) do + owner[].children[2] = replacement + return + end + keys = Parquet.StructVector(["x", "y"], [child, Int32[2]]) + owner[] = keys + maps = Parquet.MapVector(Int32[0, 1], keys, Int32[10]) + return (m=maps,) +end + +function n5hstructkeyshrink(trigger::Int) + later = Int32[2] + child = N5HActionVector(Int32[1], trigger) do + empty!(later) + return + end + keys = Parquet.StructVector(["x", "y"], [child, later]) + maps = Parquet.MapVector(Int32[0, 1], keys, Int32[10]) + return (m=maps,) +end + +function n5hstructchildthrow(error) + child = N5HActionVector(Int32[1], 1) do + throw(error) + end + return (s=Parquet.StructVector(["x", "y"], + [child, Int32[2]]),), child +end + +function n5hpublicatomic(table) + sink = IOBuffer() + Base.write(sink, UInt8[0xa5, 0x5a]) + err = n5herror() do + Parquet.write(sink, table; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test !(err isa BoundsError) + @test take!(sink) == UInt8[0xa5, 0x5a] + return err +end + +function n5hprovenancedeclaredkeyattack(value=n5hchain(64)) + bytes = Parquet._encodefile((m=Dict{Int32,Int32}[ + Dict(Int32(1) => Int32(2))],); + checksum=false, pageindex=false) + originaltable = Parquet.Table(bytes) + original = originaltable.columns.m + keys = N5HLyingInt32Vector(value, 0) + replacement = Parquet.MapVector(copy(original.offsets), keys, + original.values) + keys.calls = 0 + table = Parquet.Table(originaltable.file, originaltable.metadata, + originaltable.schema, (m=replacement,), originaltable.rows, false) + @atomic originaltable.closed = true + return table +end + +function n5hprovenancekeysequence(values::Vector{Int32}) + bytes = Parquet._encodefile((m=Dict{Int32,Int32}[ + Dict(Int32(1) => Int32(2))],); + checksum=false, pageindex=false) + originaltable = Parquet.Table(bytes) + original = originaltable.columns.m + keys = N5HSequenceVector(values, 0) + replacement = Parquet.MapVector(copy(original.offsets), keys, + collect(something(original.values)); validity=original.validity) + keys.calls = 0 + table = Parquet.Table(originaltable.file, originaltable.metadata, + originaltable.schema, (m=replacement,), originaltable.rows, false) + @atomic originaltable.closed = true + return table, keys +end + +function n5hprovenancethrowkey(error) + bytes = Parquet._encodefile((m=Dict{Int32,Int32}[ + Dict(Int32(1) => Int32(2))],); + checksum=false, pageindex=false) + originaltable = Parquet.Table(bytes) + original = originaltable.columns.m + keys = N5HActionVector(Int32[1], typemax(Int)) do + throw(error) + end + replacement = Parquet.MapVector(copy(original.offsets), keys, + original.values) + keys.calls = 0 + keys.trigger = 1 + table = Parquet.Table(originaltable.file, originaltable.metadata, + originaltable.schema, (m=replacement,), originaltable.rows, false) + @atomic originaltable.closed = true + return table, keys +end + +function n5hprovenancelistphase(trigger::Int) + bytes = Parquet._encodefile((items=[Int32[1], Int32[2]],); + checksum=false, pageindex=false) + originaltable = Parquet.Table(bytes) + original = originaltable.columns.items + owner = Ref{Any}(nothing) + child = N5HActionVector(collect(original.values), trigger) do + pop!(owner[].offsets) + return + end + replacement = Parquet.ListVector(copy(original.offsets), child; + validity=original.validity) + owner[] = replacement + table = Parquet.Table(originaltable.file, originaltable.metadata, + originaltable.schema, (items=replacement,), originaltable.rows, false) + @atomic originaltable.closed = true + return table, child +end + +function n5hprovenancestructphase(trigger::Int) + bytes = Parquet._encodefile( + (s=[(x=Int32(1), y=Int32(2))],); + checksum=false, pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + firstchild = N5HActionVector(collect(column.children[1]), trigger) do + pop!(column.children) + return + end + column.children[1] = firstchild + return table, firstchild +end + +function n5hprovenancestructsubstitution(trigger::Int) + bytes = Parquet._encodefile( + (s=[(x=Int32(1), y=Int32(2))],); + checksum=false, pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + firstchild = N5HActionVector(collect(column.children[1]), trigger) do + column.children[2] = Int32[] + return + end + column.children[1] = firstchild + return table, firstchild +end + +function n5hprovenancestructidentity(trigger::Int) + bytes = Parquet._encodefile( + (s=[(x=Int32(1), y=Int32(2))],); + checksum=false, pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + replacement = N5HThrowVector{Int32}(N5HSentinelError()) + firstchild = N5HActionVector(collect(column.children[1]), trigger) do + column.children[2] = replacement + return + end + column.children[1] = firstchild + return table, firstchild +end + +function n5hprovenancestructshrink(trigger::Int) + bytes = Parquet._encodefile( + (s=[(x=Int32(1), y=Int32(2))],); + checksum=false, pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + later = column.children[2] + firstchild = N5HActionVector(collect(column.children[1]), trigger) do + empty!(later) + return + end + column.children[1] = firstchild + return table, firstchild +end + +function n5hprovenancenestedlist(trigger::Int) + bytes = Parquet._encodefile( + (s=[(x=Int32(1), items=Int32[2])],); + checksum=false, pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + nested = column.children[2] + firstchild = N5HActionVector(collect(column.children[1]), trigger) do + nested.offsets[2] = 0 + return + end + column.children[1] = firstchild + return table, firstchild +end + +function n5hprovenancestructmapkeyidentity() + rows = [Dict((x=Int32(1), y=Int32(2)) => Int32(3))] + bytes = Parquet._encodefile((m=rows,); + checksum=false, pageindex=false) + table = Parquet.Table(bytes) + keys = table.columns.m.keys + replacement = N5HThrowVector{Int32}(N5HSentinelError()) + firstchild = N5HActionVector(collect(keys.children[1]), 1) do + keys.children[2] = replacement + return + end + keys.children[1] = firstchild + return table, firstchild +end + +function n5hprovenancemapphase(trigger::Int) + bytes = Parquet._encodefile((m=Dict{Int32,Int32}[ + Dict(Int32(1) => Int32(2))],); + checksum=false, pageindex=false) + originaltable = Parquet.Table(bytes) + original = originaltable.columns.m + owner = Ref{Any}(nothing) + keys = N5HActionVector(collect(original.keys), typemax(Int)) do + pop!(something(owner[].values)) + return + end + replacement = Parquet.MapVector(copy(original.offsets), keys, + collect(something(original.values)); validity=original.validity) + owner[] = replacement + keys.calls = 0 + keys.trigger = trigger + table = Parquet.Table(originaltable.file, originaltable.metadata, + originaltable.schema, (m=replacement,), originaltable.rows, false) + @atomic originaltable.closed = true + return table, keys +end + +function n5hprovenancescalaraxis(trigger::Int) + bytes = Parquet._encodefile((x=Int32[1, 2],); + checksum=false, pageindex=false) + originaltable = Parquet.Table(bytes) + values = N5HBackingAxisVector(collect(originaltable.columns.x), 0, + trigger, false) + table = Parquet.Table(originaltable.file, originaltable.metadata, + originaltable.schema, (x=values,), originaltable.rows, false) + @atomic originaltable.closed = true + return table, values +end + +function n5hprovenancemapaxis(trigger::Int) + bytes = Parquet._encodefile((m=Dict{Int32,Int32}[ + Dict(Int32(1) => Int32(2))],); + checksum=false, pageindex=false) + originaltable = Parquet.Table(bytes) + original = originaltable.columns.m + values = N5HBackingAxisVector(collect(something(original.values)), 0, + typemax(Int), false) + keys = N5HActionVector(collect(original.keys), typemax(Int)) do + values.shifted = true + return + end + replacement = Parquet.MapVector(copy(original.offsets), keys, values; + validity=original.validity) + keys.calls = 0 + keys.trigger = trigger + table = Parquet.Table(originaltable.file, originaltable.metadata, + originaltable.schema, (m=replacement,), originaltable.rows, false) + @atomic originaltable.closed = true + return table, keys +end + +function n5hchildsubstitution() + owner = Ref{Any}(nothing) + replacement = Int32[99] + child = N5HActionVector(Int32[7], 1) do + owner[].children[1] = replacement + return + end + column = Parquet.StructVector(["x"], [child]) + owner[] = column + return (s=column,) +end + +function n5hoffsetmutation() + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1, 2], 1) do + owner[].offsets[2] = 0 + return + end + column = Parquet.ListVector(Int32[0, 1, 2], child) + owner[] = column + return (items=column,) +end + +function n5hvaliditymutation() + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1], 1) do + owner[].validity[2] = true + return + end + column = Parquet.ListVector(Int32[0, 1, 1], child; + validity=Bool[true, false]) + owner[] = column + return (items=column,) +end + +function n5hrankmutation() + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1, 2], 1) do + owner[].ranks[2] = 0 + owner[].ranks[3] = 1 + return + end + column = Parquet.StructVector(["x"], [child]; + ranks=Int32[0, 1, 1, 2]) + owner[] = column + return (s=column,) +end + +function n5hlengthmutation() + owner = Ref{Any}(nothing) + values = N5HActionVector(Int32[1], 1) do + push!(owner[].values, Int32(2)) + return + end + owner[] = values + return (x=values,) +end + +function n5hstructnamemutation() + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1], 1) do + owner[].names[1] = "renamed" + return + end + column = Parquet.StructVector(["x"], [child]) + owner[] = column + return (s=column,) +end + +function n5hstructordermutation() + owner = Ref{Any}(nothing) + firstchild = N5HActionVector(Int32[1], 1) do + reverse!(owner[].children) + return + end + column = Parquet.StructVector(["x", "y"], + [firstchild, Int32[2]]) + owner[] = column + return (s=column,) +end + +function n5hstructcountmutation() + owner = Ref{Any}(nothing) + secondchild = N5HActionVector(Int32[2], 1) do + pop!(owner[].children) + return + end + column = Parquet.StructVector(["x", "y"], + [Int32[1], secondchild]) + owner[] = column + return (s=column,) +end + +function n5hterminaloffsetmutation() + owner = Ref{Any}(nothing) + child = N5HActionVector(Int32[1, 2], 2) do + owner[].offsets[end] = 1 + return + end + column = Parquet.ListVector(Int32[0, 2], child) + owner[] = column + return (items=column,) +end + +function n5hassertprivatefailure(table; limits::Parquet.Limits=Parquet.Limits()) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields(table, limits, budget) + end + @test err isa ArgumentError + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + return err +end + +function n5hprovenanceattack() + bytes = Parquet._encodefile((s=[(x=Int32(7),)],); checksum=false, + pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + original = column.children[1] + replacement = Int32[99] + attack = N5HActionVector(collect(original), 2) do + column.children[1] = replacement + return + end + column.children[1] = attack + return table +end + +function n5hschemaelement(element::Parquet.Metadata.SchemaElement; + type_=element.type_, type_length=element.type_length, + repetition_type=element.repetition_type, name=element.name, + num_children=element.num_children, + converted_type=element.converted_type, scale=element.scale, + precision=element.precision, field_id=element.field_id, + logicalType=element.logicalType, + unknown_fields=element.unknown_fields) + return Parquet.Metadata.SchemaElement(; type_, type_length, + repetition_type, name, num_children, converted_type, scale, precision, + field_id, logicalType, unknown_fields) +end + +function n5hprovenanceannotationattack() + date = Dates.Date(2024, 1, 2) + bytes = Parquet._encodefile((s=[(d=date,)],); checksum=false, + pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + leafindex = something(findfirst(element -> element.name == "d", + table.metadata.schema)) + original = column.children[1] + attack = N5HActionVector(collect(original), 2) do + element = table.metadata.schema[leafindex] + table.metadata.schema[leafindex] = n5hschemaelement(element; + logicalType=nothing) + return + end + column.children[1] = attack + return table +end + +function n5hprovenancerowattack() + bytes = Parquet._encodefile((s=[(x=Int32(7),)],); checksum=false, + pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + original = column.children[1] + attack = N5HActionVector(collect(original), 2) do + table.rows += 1 + return + end + column.children[1] = attack + return table +end + +function n5hprovenancetopologyattack() + bytes = Parquet._encodefile((s=[(x=Int32(7), y=Int32(8))],); + checksum=false, pageindex=false) + table = Parquet.Table(bytes) + column = table.columns.s + original = column.children[2] + attack = N5HActionVector(collect(original), 2) do + pop!(column.children) + return + end + column.children[2] = attack + return table +end + +@testset "N5-C ordinary source mutation barriers" begin + @test n5hassertprivatefailure(n5hchildsubstitution()) isa ArgumentError + @test n5hassertprivatefailure(n5hoffsetmutation()) isa ArgumentError + @test n5hassertprivatefailure(n5hvaliditymutation()) isa ArgumentError + @test n5hassertprivatefailure(n5hrankmutation()) isa ArgumentError + lengtherror = n5hassertprivatefailure(n5hlengthmutation()) + @test occursin("vector length", sprint(showerror, lengtherror)) + nameerror = n5hassertprivatefailure(n5hstructnamemutation()) + @test occursin("struct names", sprint(showerror, nameerror)) + childordererror = n5hassertprivatefailure(n5hstructordermutation()) + @test occursin("child identity or order", sprint(showerror, childordererror)) + childcounterror = n5hassertprivatefailure(n5hstructcountmutation()) + @test occursin("child identity or order", sprint(showerror, childcounterror)) + terminalerror = n5hassertprivatefailure(n5hterminaloffsetmutation()) + @test occursin("LIST offsets", sprint(showerror, terminalerror)) + + axeserror = n5hassertprivatefailure((x=N5HAxisVector(Int32[1], false),)) + @test occursin("axes", sprint(showerror, axeserror)) + + ordererror = n5hassertprivatefailure( + (m=N5HAlternatingDict[N5HAlternatingDict(0)],)) + @test occursin("MAP keys", sprint(showerror, ordererror)) + + lists = N5HSequenceVector( + [Int32[1], Int32[1, 2], Int32[1, 2]], 0) + listerror = n5hassertprivatefailure((items=lists,)) + @test occursin("occurrence", sprint(showerror, listerror)) + + maps = N5HSequenceVector( + N5HCountDict[N5HCountDict(1), N5HCountDict(2), N5HCountDict(2)], 0) + maperror = n5hassertprivatefailure((items=maps,)) + @test occursin("occurrence", sprint(showerror, maperror)) + + payload = N5HSequenceVector(["x", "x", "longer"], 0) + payloaderror = n5hassertprivatefailure((x=payload,)) + @test occursin("occurrence", sprint(showerror, payloaderror)) + + decimals = N5HSequenceVector([ + Parquet.Decimal(9, 1), Parquet.Decimal(9, 1), + Parquet.Decimal(99, 2)], 0) + decimalerror = n5hassertprivatefailure((x=decimals,)) + @test occursin("occurrence", sprint(showerror, decimalerror)) + + timestamps = N5HSequenceVector([ + Parquet.Timestamp(Int64(1), :micros, false), + Parquet.Timestamp(Int64(2), :micros, false), + Parquet.Timestamp(Int64(3), :micros, true)], 0) + timestamperror = n5hassertprivatefailure((x=timestamps,)) + @test occursin("occurrence", sprint(showerror, timestamperror)) + + lowpage = Parquet.Limits(max_page_bytes=0) + precedence = n5hassertprivatefailure(n5hchildsubstitution(); + limits=lowpage) + @test precedence isa ArgumentError +end + +@testset "N5-C live Tables column barriers" begin + source = Int32[1] + replacement = Int32[2] + table = N5HLiveTable([:x], AbstractVector[source], replacement, 0, 2) + err = n5herror() do + Parquet._encodefile(table; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("identity", sprint(showerror, err)) + + order = N5HNameOrderTable([:a, :b], + AbstractVector[Int32[1], Int32[2]], 0) + err = n5herror() do + Parquet._encodefile(order; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("name or order", sprint(showerror, err)) + + rename = N5HRenameTable(:a, Int32[1], 0) + err = n5herror() do + Parquet._encodefile(rename; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("name or order", sprint(showerror, err)) + + badnames = N5HBadNamesTable(0, Int32[1], Int32[2]) + err = n5herror() do + Parquet._encodefile(badnames; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test !(err isa BoundsError) +end + +@testset "N5-C provenance late mutation barrier" begin + attacks = ( + (n5hprovenanceattack, "child identity"), + (n5hprovenanceannotationattack, "SchemaElement"), + (n5hprovenancerowattack, "row count"), + (n5hprovenancetopologyattack, "child count"), + ) + for (factory, message) in attacks + table = factory() + try + budget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields(table, Parquet.Limits(), budget) + end + @test err isa ArgumentError + @test occursin(message, sprint(showerror, err)) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + finally + close(table) + end + end +end + +@testset "N5-C controls and recursive key copies" begin + changing = N5HSequenceVector(Int32[1, 2, 3], 0) + fields, rows = Parquet._writefields((x=changing,), Parquet.Limits()) + @test rows == 1 + @test only(only(fields).leaves).values == Int32[3] + @test changing.calls == 3 + + zeroaxis = N5HZeroAxisVector(Int32[4, 5]) + bytes = Parquet._encodefile((x=zeroaxis,); checksum=false, + pageindex=false) + table = Parquet.Table(bytes) + try + @test collect(table.columns.x) == Int32[4, 5] + finally + close(table) + end + + keys = Parquet.StructVector(["optional"], + [Union{Missing,Int32}[missing, Int32(2)]]) + maps = Parquet.MapVector(Int32[0, 2], keys, Int32[10, 20]) + for version in (:v1, :v2) + bytes = Parquet._encodefile((m=maps,); checksum=false, + pageversion=version, pageindex=false) + @test bytes == Parquet._encodefile((m=maps,); checksum=false, + pageversion=version, pageindex=false) + table = Parquet.Table(bytes) + try + pairs = collect(table.columns.m[1]) + @test length(pairs) == 2 + @test pairs[1].first[1] === missing + @test pairs[2].first[1] == Int32(2) + finally + close(table) + end + end + + duplicate = Parquet.MapVector(Int32[0, 2], Int32[1, 1], + Int32[10, 11]) + bytes = Parquet._encodefile((m=duplicate,); checksum=false, + pageindex=false) + table = Parquet.Table(bytes) + try + @test collect(table.columns.m[1]) == + [Int32(1) => Int32(10), Int32(1) => Int32(11)] + finally + close(table) + end + +end + +@testset "N5-C bounded authoritative MAP-key snapshots" begin + limits = Parquet.Limits(max_metadata_depth=4) + lying = (m=N5HLyingCycleDict[N5HLyingCycleDict()],) + err = n5herror() do + Parquet._encodefile(lying; checksum=false, pageindex=false, + limits=limits) + end + @test err isa ArgumentError + @test !(err isa StackOverflowError) + @test occursin("expected Int32", sprint(showerror, err)) + + sink = IOBuffer() + Base.write(sink, UInt8[0xa5, 0x5a]) + err = n5herror() do + Parquet.write(sink, lying; checksum=false, pageindex=false, + limits=limits) + end + @test err isa ArgumentError + @test take!(sink) == UInt8[0xa5, 0x5a] + + budget = Parquet._LiveByteBudget(Parquet.Limits( + max_materialized_bytes=100_000, max_metadata_depth=4)) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, + [[[Int32(1)]]], nothing, Parquet.Limits(max_metadata_depth=4)) + @test snapshot isa Parquet._NestedWriteKeySnapshot + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_metadata_depth=3) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, [[[Int32(1)]]], nothing, + limits) + end + @test err isa Parquet.LimitError + @test err.resource == :metadata_depth + @test err.requested == 4 + @test err.maximum == 3 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=192, + max_metadata_depth=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, Int32[1], nothing, limits) + end + @test err isa Parquet.LimitError + @test err.resource == :metadata_depth + @test err.requested == 2 + @test err.maximum == 1 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + value = N5HSameSourceString(N5HSequencedCodeunitVector(0)) + limits = Parquet.Limits(max_materialized_bytes=192, + max_string_bytes=0) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, value, nothing, limits) + end + @test err isa Parquet.LimitError + @test err.resource == :string_bytes + @test value.bytes.reads == 0 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, value, nothing, limits) + end + @test err isa ArgumentError + @test occursin("invalid UTF-8", sprint(showerror, err)) + @test value.bytes.reads == 1 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_metadata_depth=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, Int32[], nothing, limits) + @test isempty(snapshot.children) + Parquet._nestedwritetracecompare!(trace) + @test Parquet._nestedwritetracekey!(trace, Int32[], nothing, limits) === + snapshot + Parquet._nestedwritetracefinishcompare!(trace) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + capturelimits = Parquet.Limits(max_materialized_bytes=100_000, + max_metadata_depth=2) + budget = Parquet._LiveByteBudget(capturelimits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + Parquet._nestedwritetracekey!(trace, Int32[1], nothing, capturelimits) + Parquet._nestedwritetracecompare!(trace) + before = Parquet._budgetused(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, Int32[1], nothing, + Parquet.Limits(max_metadata_depth=1)) + end + @test err isa Parquet.LimitError + @test err.resource == :metadata_depth + @test Parquet._budgetused(budget) == before + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + cycle = [] + push!(cycle, cycle) + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_metadata_depth=4) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, cycle, nothing, limits) + end + @test err isa ArgumentError + @test occursin("cyclic", sprint(showerror, err)) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + value = N5HSequencedString(0) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, value, nothing, limits) + @test snapshot.value == UInt8[0x61] + @test value.calls == 1 + Parquet._nestedwritetracecompare!(trace) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, value, nothing, limits) + end + @test err isa ArgumentError + @test value.calls == 2 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_metadata_depth=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, cycle, nothing, limits) + end + @test err isa Parquet.LimitError + @test err.resource == :metadata_depth + @test err.requested == 2 + @test err.maximum == 1 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=8_192, + max_metadata_depth=1_000_000) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, Int32(1), nothing, + limits) + @test snapshot.value == Int32(1) + @test Parquet._budgetused(budget) <= limits.max_materialized_bytes + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_container_elements=2) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, Int32[1, 2], nothing, + limits) + @test length(snapshot.children) == 2 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=192, + max_container_elements=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, Int32[1, 2], nothing, limits) + end + @test err isa Parquet.LimitError + @test err.resource == :container_elements + @test err.requested == 2 + @test err.maximum == 1 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_container_elements=0, max_string_bytes=2) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, + (UInt8(0x61), UInt8(0x62)), nothing, limits) + @test snapshot.value == UInt8[0x61, 0x62] + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_string_bytes=2) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, UInt8[0x61, 0x62], + nothing, limits) + @test snapshot.value == UInt8[0x61, 0x62] + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=192, + max_string_bytes=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, UInt8[0x61, 0x62], nothing, + limits) + end + @test err isa Parquet.LimitError + @test err.resource == :string_bytes + @test err.requested == 2 + @test err.maximum == 1 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + aggregate = Parquet._NestedWriteLeafAggregate(false, nothing, nothing, + Int32(1)) + shape = Parquet._NestedWriteLeafShape("key", Vector{UInt8}, false, + nothing, Int32(2), aggregate) + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_container_elements=0, max_string_bytes=2) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, UInt8[0x61, 0x62], + shape, limits) + @test snapshot.value == UInt8[0x61, 0x62] + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=192, + max_container_elements=0, max_string_bytes=2) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, UInt8[0x61], shape, limits) + end + @test err isa ArgumentError + @test occursin("wrong width", sprint(showerror, err)) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + decimal = Parquet.Decimal(1234567890123456789, 0) + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_decimal_bytes=9) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, decimal, nothing, limits) + @test snapshot.value == decimal + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=192, + max_decimal_bytes=8) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, decimal, nothing, limits) + end + @test err isa Parquet.LimitError + @test err.resource == :decimal_bytes + @test err.requested == 9 + @test err.maximum == 8 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_decimal_bytes=0) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, + Parquet.Decimal(12, 0), nothing, limits) + @test snapshot.value == Parquet.Decimal(12, 0) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) +end + +@testset "N5-C MAP-key conversion and consume gap" begin + limits = Parquet.Limits(max_materialized_bytes=100_000) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, N5HBadByteKey(), nothing, + limits) + end + @test err isa ArgumentError + @test !(err isa InexactError) + @test occursin("not a UInt8", sprint(showerror, err)) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, N5HAxisShiftByteKey(false), + nothing, limits) + end + @test err isa ArgumentError + @test occursin("axes", sprint(showerror, err)) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + key = N5HShiftBadByteKey(false) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, key, nothing, limits) + end + @test err isa ArgumentError + @test occursin("axes", sprint(showerror, err)) + @test !occursin("not a UInt8", sprint(showerror, err)) + @test key.shifted + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + key = N5HSequencedShiftBadByteKey(0, 2, false) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + Parquet._nestedwritetracekey!(trace, key, nothing, limits) + Parquet._nestedwritetracecompare!(trace) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, key, nothing, limits) + end + @test err isa ArgumentError + @test occursin("axes", sprint(showerror, err)) + @test !occursin("not a UInt8", sprint(showerror, err)) + @test key.calls == 2 + @test key.shifted + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + sequenced = N5HSequencedLengthByteKey(0) + limits = Parquet.Limits(max_materialized_bytes=100_000, + max_string_bytes=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, sequenced, nothing, limits) + end + @test err isa ArgumentError + @test !(err isa BoundsError) + @test occursin("length", sprint(showerror, err)) + @test sequenced.calls == 1 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + sentinel = N5HSentinelError() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, N5HThrowLengthKey(sentinel), + nothing, limits) + end + @test err === sentinel + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + sentinel = N5HSentinelError() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, N5HThrowByteKey(sentinel), + nothing, limits) + end + @test err === sentinel + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + key = N5HSequencedByteKey(0, 6) + maps = Parquet.MapVector(Int32[0, 1], N5HSequencedByteKey[key], + Int32[7]) + err = n5herror() do + Parquet._encodefile((m=maps,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("physically consumed", sprint(showerror, err)) + @test key.passes == 6 + + key = N5HSequencedByteKey(0, 6) + maps = Parquet.MapVector(Int32[0, 1], N5HSequencedByteKey[key], + Int32[7]) + sink = IOBuffer() + Base.write(sink, UInt8[0xde, 0xad]) + err = n5herror() do + Parquet.write(sink, (m=maps,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test take!(sink) == UInt8[0xde, 0xad] + + key = N5HSequencedShiftBadByteKey(0, 6, false) + maps = Parquet.MapVector(Int32[0, 1], + N5HSequencedShiftBadByteKey[key], Int32[7]) + err = n5herror() do + Parquet._encodefile((m=maps,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("axes", sprint(showerror, err)) + @test !occursin("not a UInt8", sprint(showerror, err)) + @test key.calls == 6 + @test key.shifted + + key = N5HSequencedShiftBadByteKey(0, 4, false) + maps = Parquet.MapVector(Int32[0, 1], + N5HShiftBadString[N5HShiftBadString(key)], Int32[7]) + err = n5herror() do + Parquet._encodefile((m=maps,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("axes", sprint(showerror, err)) + @test !occursin("not a UInt8", sprint(showerror, err)) + @test !occursin("UTF-8", sprint(showerror, err)) + @test key.calls == 4 + @test key.shifted + + key = N5HSequencedShiftBadByteKey(0, 6, false) + maps = Parquet.MapVector(Int32[0, 1], + N5HShiftBadString[N5HShiftBadString(key)], Int32[7]) + err = n5herror() do + Parquet._encodefile((m=maps,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("axes", sprint(showerror, err)) + @test !occursin("not a UInt8", sprint(showerror, err)) + @test !occursin("UTF-8", sprint(showerror, err)) + @test key.calls == 6 + @test key.shifted + + for attack in 1:3 + value = N5HNCodeunitAxisString(N5HNCodeunitAxisBytes(false), 0, + attack) + maps = Parquet.MapVector(Int32[0, 1], + N5HNCodeunitAxisString[value], Int32[7]) + err = n5herror() do + Parquet._encodefile((m=maps,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("axes", sprint(showerror, err)) + @test value.calls == attack + @test value.bytes.shifted + end + + key = N5HSequencedByteKey(0, typemax(Int)) + maps = Parquet.MapVector(Int32[0, 1], N5HSequencedByteKey[key], + Int32[7]) + bytes = Parquet._encodefile((m=maps,); checksum=false, pageindex=false) + @test key.passes == 7 + table = Parquet.Table(bytes) + try + pair = only(collect(table.columns.m[1])) + @test pair.first == UInt8[0x61] + @test pair.second == Int32(7) + finally + close(table) + end + + decimal = Parquet.Decimal(12, 0) + decimalmap = Parquet.MapVector(Int32[0, 1], + Parquet.Decimal[decimal], Int32[1]) + fields, rows = Parquet._writefields((m=decimalmap,), Parquet.Limits()) + @test rows == 1 + element = fields[1].schema[3] + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + expected = Parquet._nestedwritetracekey!(trace, decimal, nothing, limits) + physical = try + Base.GMP.MPZ.set!(decimal.unscaled, BigInt(13)) + Parquet._nestedwritenormalizekeyphysical(element, decimal, limits) + finally + Base.GMP.MPZ.set!(decimal.unscaled, BigInt(12)) + end + context = Parquet._NestedWriteEmitContext( + Parquet._NestedWriteLeafBuilder[], Parquet._NestedWriteLeafCount[], + limits, nothing, trace) + @test !Parquet._nestedwritekeyphysicalequal(expected, physical, element, + context) + @test decimal == Parquet.Decimal(12, 0) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) +end + +@testset "N5-C budget ownership and exception identity" begin + limits = Parquet.Limits(max_materialized_bytes=65) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + names = ["x"] + values = AbstractVector[Int32[1]] + columns = Pair{String,AbstractVector}["x" => values[1]] + err = n5herror() do + Parquet._nestedwritetopology(columns, names, values, limits, budget) + end + @test err isa Parquet.LimitError + @test err.resource == :materialized_bytes + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=100_000) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + topology = Parquet._nestedwritetopology(columns, names, values, limits, + budget) + @test Parquet._budgetused(budget) > 64 + Parquet._nestedwritetopologyrelease!(topology, budget) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=400) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, repeat("x", 1024)) + end + @test err isa Parquet.LimitError + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + limits = Parquet.Limits(max_materialized_bytes=1_024, + max_container_elements=typemax(Int64)) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, N5HCountDict(typemax(Int)), + nothing, limits) + end + @test err isa Parquet.LimitError + @test err.resource == :materialized_bytes + @test err.requested > limits.max_materialized_bytes + @test err.maximum == limits.max_materialized_bytes + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + sentinel = N5HSentinelError() + budget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields((x=N5HThrowVector{Int32}(sentinel),), + Parquet.Limits(), budget) + end + @test err === sentinel + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) +end + +@testset "N5-C public sink and path atomicity" begin + for version in (:v1, :v2) + sink = IOBuffer() + Base.write(sink, UInt8[0xa5, 0x5a]) + err = n5herror() do + Parquet.write(sink, n5hchildsubstitution(); pageversion=version, + checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test take!(sink) == UInt8[0xa5, 0x5a] + end + + mktempdir() do directory + path = joinpath(directory, "atomic.parquet") + sentinel = UInt8[0xde, 0xad, 0xbe, 0xef] + Base.write(path, sentinel) + err = n5herror() do + Parquet.write(path, n5hchildsubstitution(); checksum=false, + pageindex=false) + end + @test err isa ArgumentError + @test Base.read(path) == sentinel + return + end +end + +@testset "N5-C iterative deep MAP-key frames" begin + depth = 20_000 + value = n5hchain(depth) + limits = Parquet.Limits(max_materialized_bytes=32_000_000, + max_metadata_depth=depth + 1, max_container_elements=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, value, nothing, limits) + @test snapshot isa Parquet._NestedWriteKeySnapshot + Parquet._nestedwritetracecompare!(trace) + @test Parquet._nestedwritetracekey!(trace, value, nothing, limits) === + snapshot + Parquet._nestedwritetracefinishcompare!(trace) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + boundary = n5hchain(64) + exact = Parquet.Limits(max_materialized_bytes=1_000_000, + max_metadata_depth=65, max_container_elements=1) + budget = Parquet._LiveByteBudget(exact) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + Parquet._nestedwritetracekey!(trace, boundary, nothing, exact) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + oneover = Parquet.Limits(max_materialized_bytes=1_000_000, + max_metadata_depth=64, max_container_elements=1) + budget = Parquet._LiveByteBudget(oneover) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, boundary, nothing, oneover) + end + @test err isa Parquet.LimitError + @test err.resource == :metadata_depth + @test err.requested == 65 + @test err.maximum == 64 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + cycle = n5hcycle(64) + cyclelimits = Parquet.Limits(max_materialized_bytes=1_000_000, + max_metadata_depth=66, max_container_elements=1) + budget = Parquet._LiveByteBudget(cyclelimits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, cycle, nothing, cyclelimits) + end + @test err isa ArgumentError + @test !(err isa StackOverflowError) + @test occursin("cyclic", sprint(showerror, err)) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + cycleoneover = Parquet.Limits(max_materialized_bytes=1_000_000, + max_metadata_depth=65, max_container_elements=1) + budget = Parquet._LiveByteBudget(cycleoneover) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, cycle, nothing, cycleoneover) + end + @test err isa Parquet.LimitError + @test err.resource == :metadata_depth + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + mismatch = n5hchain(64, Int32(2)) + budget = Parquet._LiveByteBudget(exact) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + Parquet._nestedwritetracekey!(trace, boundary, nothing, exact) + Parquet._nestedwritetracecompare!(trace) + before = Parquet._budgetused(budget) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, mismatch, nothing, exact) + end + @test err isa ArgumentError + @test Parquet._budgetused(budget) == before + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + shallow = Parquet.Limits(max_materialized_bytes=8_192, + max_metadata_depth=1_000_000, max_container_elements=1) + budget = Parquet._LiveByteBudget(shallow) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + snapshot = Parquet._nestedwritetracekey!(trace, N5HChain(Int32(1)), + nothing, shallow) + @test only(snapshot.children).value == Int32(1) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + for sentinel in (N5HMutableSentinel(1), BoundsError(:user, 7)) + budget = Parquet._LiveByteBudget(exact) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + hostile = N5HChain(N5HThrowAnyVector(sentinel)) + err = n5herror() do + Parquet._nestedwritetracekey!(trace, hostile, nothing, exact) + end + @test err === sentinel + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + end +end + +@testset "N5-C stable package validators" begin + for (factory, needle) in ( + (() -> begin + owner = Ref{Any}(nothing) + child = N5HLengthMutationVector(Int32[11], () -> begin + owner[].offsets[2] = 0 + return + end, false, 0) + column = Parquet.ListVector(Int32[0, 1], child) + owner[] = column + return column, child + end, "LIST offsets"), + (() -> begin + owner = Ref{Any}(nothing) + keys = N5HLengthMutationVector(Int32[1], () -> begin + owner[].offsets[2] = 0 + return + end, false, 0) + column = Parquet.MapVector(Int32[0, 1], keys, Int32[11]) + owner[] = column + return column, keys + end, "MAP offsets"), + (() -> begin + owner = Ref{Any}(nothing) + child = N5HLengthMutationVector(Int32[11], () -> begin + owner[].ranks[2] = 0 + return + end, false, 0) + column = Parquet.StructVector(["x"], [child]; + ranks=Int32[0, 1]) + owner[] = column + return column, child + end, "struct ranks"), + ) + column, child = factory() + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + topology = Parquet._nestedwritetopology(nothing, ["x"], + AbstractVector[column], limits, budget) + try + snapshot = Parquet._nestedwritesnapshotfor(topology, column) + child.calls = 0 + child.armed = true + err = n5herror() do + Parquet._nestedwritevalidatenode(snapshot) + end + @test err isa ArgumentError + @test occursin(needle, sprint(showerror, err)) + @test child.calls == 1 + finally + Parquet._nestedwritetopologyrelease!(topology, budget) + end + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + @test Parquet._budgetused(budget) == 0 + end + + for (factory, validate, needle) in ( + (() -> begin + owner = Ref{Any}(nothing) + child = N5HLengthMutationVector(Int32[11], () -> begin + owner[].offsets[1] = 1 + return + end, false, 0) + column = Parquet.ListVector(Int32[0, 1], child) + owner[] = column + return column, child + end, Parquet._validatelistvector, "list offsets"), + (() -> begin + owner = Ref{Any}(nothing) + keys = N5HLengthMutationVector(Int32[1], () -> begin + owner[].offsets[1] = 1 + return + end, false, 0) + column = Parquet.MapVector(Int32[0, 1], keys, Int32[11]) + owner[] = column + return column, keys + end, Parquet._validatemapvector, "map offsets"), + (() -> begin + owner = Ref{Any}(nothing) + child = N5HLengthMutationVector(Int32[11], () -> begin + owner[].ranks[1] = 1 + return + end, false, 0) + column = Parquet.StructVector(["x"], [child]; + ranks=Int32[0, 1]) + owner[] = column + return column, child + end, Parquet._validatestructvector, "struct ranks"), + ) + column, child = factory() + child.armed = true + err = n5herror() do + validate(column) + end + @test err isa ArgumentError + @test occursin(needle, sprint(showerror, err)) + @test child.calls == 1 + end + + names = ["x"] + child = N5HLengthMutationVector(Int32[11], () -> begin + empty!(names) + return + end, false, 0) + value = Parquet.StructValue(names, AbstractVector[child], 1) + child.armed = true + err = n5herror() do + Parquet._validatestructvalue(value) + end + @test err isa ArgumentError + @test occursin("struct child identity", sprint(showerror, err)) + @test child.calls == 1 + + keys = Int32[1] + values = N5HLengthMutationVector(Int32[11], () -> begin + empty!(keys) + return + end, false, 0) + value = Parquet.MapValue{Int32,Int32,true}(keys, values, 1, 1) + values.armed = true + err = n5herror() do + Parquet._validatemapvalue(value) + end + @test err isa ArgumentError + @test occursin("map view", sprint(showerror, err)) + @test values.calls == 1 +end + +@testset "N5-C within-pass package mutation" begin + stable = N5HNothingStateDict(Int32(1) => Int32(2), 0, 0) + materialization = Parquet._nestedwritedictmaterialize(nothing, stable, + nothing, nothing, Parquet.Limits()) + @test materialization.count == 1 + @test materialization.first.key == Int32(1) + @test materialization.first.value == Int32(2) + @test materialization.first.next === nothing + @test stable.calls == 2 + @test stable.length_calls == 0 + Parquet._nestedwritedictrelease!(nothing, materialization) + + for invalid in ( + Parquet.ListValue(N5HThrowMetricChild( + N5HMutableSentinel(81)), 0, -1), + Parquet.MapValue{Int32,Int32,:bad}( + N5HThrowMetricChild(N5HMutableSentinel(82)), nothing, 1, 0), + ) + dictionary = Dict{Int32,Any}(Int32(1) => invalid) + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritedictmaterialize(trace, dictionary, nothing, + nothing, limits) + end + @test err isa ArgumentError + @test !(err isa N5HMutableSentinel) + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + end + + for reverse in (false, true) + sentinel = N5HMutableSentinel(reverse ? 84 : 83) + callback = Parquet.ListValue(N5HThrowMetricChild(sentinel), 1, 0) + malformed = Parquet.ListVector(Int32[0, 1], Int32[11]) + malformed.offsets[1] = 1 + nested = Parquet.ListValue(malformed, 1, 1) + pair = reverse ? nested => callback : callback => nested + dictionary = N5HNothingStateDict(pair, 0, 0) + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritedictmaterialize(trace, dictionary, nothing, + nothing, limits) + end + @test err isa ArgumentError + @test err !== sentinel + @test dictionary.calls == 1 + @test dictionary.length_calls == 0 + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + end + + sentinel = N5HMutableSentinel(86) + dictionary = N5HSecondTouchDict(N5HThrowMetricChild(sentinel)) + limits = Parquet.Limits(max_container_elements=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + trace = Parquet._nestedwritetrace(budget) + err = n5herror() do + Parquet._nestedwritedictmaterialize(trace, dictionary, nothing, + nothing, limits) + end + @test err isa Parquet.LimitError + @test err.resource == :container_elements + @test err !== sentinel + Parquet._nestedwritetracerelease!(trace) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + owner = Ref{Any}(nothing) + child = N5HRestoringChild(owner, true, 0) + parent = Parquet.ListVector(Int32[0, 1], child) + owner[] = parent + dictionary = N5HParentAfterChildDict(parent, 0) + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + topology = Parquet._nestedwritetopology(nothing, ["x"], + AbstractVector[Int32[1]], limits, budget) + trace = Parquet._nestedwritetrace(budget, topology) + err = n5herror() do + Parquet._nestedwritedictmaterialize(trace, dictionary, nothing, + nothing, limits) + end + @test err isa ArgumentError + @test occursin("LIST offsets", sprint(showerror, err)) + @test dictionary.attacks == 1 + @test child.restores == 0 + @test parent.offsets == Int32[0, 0] + Parquet._nestedwritetracerelease!(trace) + Parquet._nestedwritetopologyrelease!(topology, budget) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + old = Parquet.ListVector(Int32[0, 1], Int32[11]) + new = Parquet.ListVector(Int32[0, 1], Int32[11]) + sentinel = N5HMutableSentinel(85) + changing = N5HChangingSourceDict(old, new, 0, 0, sentinel) + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields((m=[changing],), limits, budget) + end + @test err isa ArgumentError + @test err !== sentinel + @test changing.starts == 2 + @test changing.terminals == 1 + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + function n5haliascalls(entries::Int) + child = N5HCountingVector(0) + parent = Parquet.ListVector(Int32[0, 1], child) + child.calls = 0 + dictionary = Dict{Int32,Any}(Int32(index) => parent for index in + 1:entries) + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + topology = Parquet._nestedwritetopology(nothing, ["x"], + AbstractVector[Int32[1]], limits, budget) + trace = Parquet._nestedwritetrace(budget, topology) + materialization = Parquet._nestedwritedictmaterialize(trace, + dictionary, nothing, nothing, limits) + calls = child.calls + @test materialization.count == entries + dependencies = 0 + dependency = materialization.dependencies + while dependency !== nothing + dependencies += 1 + dependency = dependency.next + end + @test dependencies == 2 + @test materialization.dependency_sources.count == 2 + Parquet._nestedwritedictrelease!(trace, materialization) + Parquet._nestedwritetracerelease!(trace) + Parquet._nestedwritetopologyrelease!(topology, budget) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + return calls + end + @test n5haliascalls(32) == n5haliascalls(1) + + inner = Parquet.ListVector(Int32[0, 1], Int32[11]) + dictionary = N5HFinalMutKeyDict(inner => Int32(22)) + keys = N5HRestoreVector(dictionary, dictionary, true) + outer = Parquet.MapVector(Int32[0, 1], keys, Int32[7]) + err = n5herror() do + Parquet._encodefile((m=outer,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("LIST offsets", sprint(showerror, err)) + @test dictionary.attacks == 1 + @test inner.offsets == Int32[0, 0] + + inner = Parquet.ListVector(Int32[0, 1], Int32[11]) + dictionary = N5HFinalMutMap(Int32(22) => inner) + rows = N5HRestoreVector(dictionary, dictionary, false) + err = n5herror() do + Parquet._encodefile((m=rows,); checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test occursin("LIST offsets", sprint(showerror, err)) + @test dictionary.attacks == 1 + @test inner.offsets == Int32[0, 0] + + inner = Parquet.ListVector(Int32[0, 1], Int32[11]) + dictionary = N5HFinalMutMap(Int32(22) => inner) + rows = N5HRestoreVector(dictionary, dictionary, false) + err = n5herror() do + Parquet._encodefile((a=inner, m=rows); checksum=false, + pageindex=false) + end + @test err isa ArgumentError + @test occursin("LIST offsets", sprint(showerror, err)) + @test dictionary.attacks == 1 + @test inner.offsets == Int32[0, 0] + + owner = Ref{Any}(nothing) + child = N5HRestoringChild(owner, true, 0) + inner = Parquet.ListVector(Int32[0, 1], child) + owner[] = inner + dictionary = N5HFinalMutMap(Int32(22) => inner) + err = n5herror() do + Parquet._encodefile((m=[dictionary],); checksum=false, + pageindex=false) + end + @test err isa ArgumentError + @test occursin("LIST offsets", sprint(showerror, err)) + @test dictionary.attacks == 1 + @test child.restores == 0 + @test inner.offsets == Int32[0, 0] + + cases = ( + (n5hlistphasemutation, (1, 3, 5), + table -> table.items.values.calls), + (n5hlistaxismutation, (1, 3, 5), + table -> table.items.values.calls), + (n5hstructphasemutation, (1, 2, 3), + table -> table.s.children[1].calls), + (n5hstructchildsubstitution, (1, 2, 3), + table -> table.s.children[1].calls), + (n5hstructchildidentity, (1, 2, 3), + table -> table.s.children[1].calls), + (n5hstructchildshrink, (1, 2, 3), + table -> table.s.children[1].calls), + (n5hmapphasemutation, (1, 2, 3), + table -> table.m.keys.calls), + (n5hmapaxismutation, (1, 2, 3), + table -> table.m.keys.calls), + (n5hstructkeymutation, (1, 4, 7), + table -> table.m.keys.children[1].calls), + (n5hstructkeysubstitution, (1, 4, 7), + table -> table.m.keys.children[1].calls), + (n5hstructkeyidentity, (1, 4, 7), + table -> table.m.keys.children[1].calls), + (n5hstructkeyshrink, (1, 4, 7), + table -> table.m.keys.children[1].calls), + ) + for (factory, triggers, calls) in cases + for trigger in triggers + table = factory(trigger) + err = n5hassertprivatefailure(table) + @test !(err isa BoundsError) + @test calls(table) == trigger + end + end + for (factory, trigger, calls) in ( + (n5hlistphasemutation, 5, table -> table.items.values.calls), + (n5hlistaxismutation, 5, table -> table.items.values.calls), + (n5hstructphasemutation, 3, + table -> table.s.children[1].calls), + (n5hstructchildidentity, 3, + table -> table.s.children[1].calls), + (n5hstructchildshrink, 3, + table -> table.s.children[1].calls), + (n5hstructkeyidentity, 7, + table -> table.m.keys.children[1].calls), + (n5hstructkeyshrink, 7, + table -> table.m.keys.children[1].calls), + (n5hmapaxismutation, 3, table -> table.m.keys.calls), + (n5hmapphasemutation, 3, table -> table.m.keys.calls)) + table = factory(trigger) + n5hpublicatomic(table) + @test calls(table) == trigger + end +end + +@testset "N5-C provenance declared MAP-key root" begin + table = n5hprovenancedeclaredkeyattack() + try + sink = IOBuffer() + Base.write(sink, UInt8[0xa5, 0x5a]) + err = n5herror() do + Parquet.write(sink, table; checksum=false, pageindex=false, + limits=Parquet.Limits(max_metadata_depth=4)) + end + @test err isa ArgumentError + @test !(err isa StackOverflowError) + @test occursin("expected Int32", sprint(showerror, err)) + @test table.columns.m.keys.calls == 1 + @test take!(sink) == UInt8[0xa5, 0x5a] + finally + close(table) + end +end + +@testset "N5-C provenance within-pass package mutation" begin + cases = ( + (n5hprovenancelistphase, (1, 3)), + (n5hprovenancescalaraxis, (1, 3)), + (n5hprovenancestructphase, (1, 2)), + (n5hprovenancestructsubstitution, (1, 2)), + (n5hprovenancestructidentity, (1, 2)), + (n5hprovenancestructshrink, (1, 2)), + (n5hprovenancemapphase, (1, 2)), + (n5hprovenancemapaxis, (1, 2)), + ) + for (factory, triggers) in cases + for trigger in triggers + table, callback = factory(trigger) + try + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields(table, limits, budget) + end + @test err isa ArgumentError + @test !(err isa BoundsError) + @test callback.calls == trigger + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + finally + close(table) + end + end + end + + table, callback = n5hprovenancestructmapkeyidentity() + try + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields(table, limits, budget) + end + @test err isa ArgumentError + @test !(err isa BoundsError) + @test callback.calls == 1 + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + finally + close(table) + end + + table, callback = n5hprovenancestructmapkeyidentity() + try + n5hpublicatomic(table) + @test callback.calls == 1 + finally + close(table) + end +end + +@testset "N5-C wrapper and provenance exception identity" begin + for sentinel in (N5HMutableSentinel(2), BoundsError(:wrapper, 9)) + keys = N5HActionVector(Int32[1], typemax(Int)) do + throw(sentinel) + end + map = Parquet.MapVector(Int32[0, 1], keys, Int32[10]) + keys.calls = 0 + keys.trigger = 1 + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields((m=map,), limits, budget) + end + @test err === sentinel + @test keys.calls == 1 + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + structtable, structchild = n5hstructchildthrow(sentinel) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields(structtable, limits, budget) + end + @test err === sentinel + @test structchild.calls == 1 + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + + table, provenancekeys = n5hprovenancethrowkey(sentinel) + try + sink = IOBuffer() + Base.write(sink, UInt8[0xa5, 0x5a]) + err = n5herror() do + Parquet.write(sink, table; checksum=false, pageindex=false) + end + @test err === sentinel + @test provenancekeys.calls == 1 + @test take!(sink) == UInt8[0xa5, 0x5a] + finally + close(table) + end + end +end + + +@testset "N5-C stable package invariants and hostile metrics" begin + list = Parquet.ListVector(Int32[0, 0], Int32[]; + validity=Bool[false]) + list.offsets[2] = 1 + @test n5hassertprivatefailure((items=list,)) isa ArgumentError + + map = Parquet.MapVector(Int32[0, 0], Int32[], Int32[]; + validity=Bool[false]) + map.offsets[2] = 1 + @test n5hassertprivatefailure((m=map,)) isa ArgumentError + + interior = Parquet.ListVector(Int32[0, 0, 0], Int32[]) + interior.offsets[2] = 100 + limits = Parquet.Limits(max_container_elements=10) + err = n5hassertprivatefailure((items=interior,); limits=limits) + @test err isa ArgumentError + @test !(err isa Parquet.LimitError) + + required = Parquet.StructVector(["x"], [Int32[1]]) + push!(required.children[1], Int32(2)) + @test n5hassertprivatefailure((s=required,)) isa ArgumentError + + optional = Parquet.StructVector(["x"], [Int32[1, 2]]; + ranks=Int32[0, 1, 2]) + optional.ranks[2] = 2 + @test n5hassertprivatefailure((s=optional,)) isa ArgumentError + + rawoptional = Parquet.StructVector{ + Union{Missing,Parquet.StructValue},Vector{Int32}}( + String[], Int32[0], AbstractVector[], 0) + @test_throws ArgumentError Parquet._validatestructvector(rawoptional) + + @test_throws ArgumentError Parquet._validatelistvalue( + Parquet.ListValue(Int32[], typemin(Int), typemax(Int))) + @test_throws ArgumentError Parquet._validatelistvalue( + Parquet.ListValue(Int32[], 2, 1)) + @test_throws ArgumentError Parquet._validatemapvalue( + Parquet.MapValue{Int32,Missing,false}( + Int32[], Missing[], 1, 0)) + @test_throws ArgumentError Parquet._validatemapvalue( + Parquet.MapValue{Int32,Int32,true}(Int32[], nothing, 1, 0)) + @test_throws ArgumentError Parquet._validatemapvalue( + Parquet.MapValue{Int32,Int32,:invalid}( + Int32[], Int32[], 1, 0)) + @test_throws ArgumentError Parquet._validatemapvalue( + Parquet.MapValue{Int32,Int32,false}(Int32[], nothing, 1, 0)) + @test_throws ArgumentError Parquet._validatemapvalue( + Parquet.MapValue{Union{Missing,Int32},Int32,true}( + Union{Missing,Int32}[], Int32[], 1, 0)) + + @test_throws ArgumentError Parquet.ListVector(Int32[0], + N5HZeroAxisVector(Int32[])) + @test_throws ArgumentError Parquet.MapVector(Int32[0], + N5HZeroAxisVector(Int32[]), N5HZeroAxisVector(Int32[])) + + rawmap = Parquet.MapVector{ + Parquet.MapValue{Int32,Int32,:invalid},Int32,Int32,:invalid, + Int32,Nothing}(Int32[0], nothing, Int32[], Int32[]) + @test_throws ArgumentError Parquet._validatemapvector(rawmap) + + for hostile in (N5HNonIntLengthVector(), N5HUIntSizeVector(), + N5HUIntAxisVector()) + err = n5herror() do + Parquet._nestedvectorcount(hostile, "hostile vector") + Parquet._nestedvectoraxes(hostile, "hostile vector") + end + @test err isa ArgumentError + @test !(err isa Union{MethodError,InexactError}) + end + err = n5herror() do + Parquet._nestedwritekeycount(N5HHugeUIntDict()) + end + @test err isa ArgumentError + @test !(err isa Union{MethodError,InexactError}) + + huge = N5HHugeVector() + @test length(Parquet.ListValue(huge, typemax(Int), + typemax(Int) - 1)) == 0 + @test_throws ArgumentError Parquet._nestedspan( + Int64[typemax(Int), typemax(Int)], 1) + + for sentinel in (N5HMutableSentinel(71), BoundsError(:metric, 3)) + @test n5herror(() -> Parquet._nestedvectorcount( + N5HThrowSizeVector(sentinel), "hostile vector")) === sentinel + @test n5herror(() -> Parquet._nestedvectorcount( + N5HThrowLengthKey(sentinel), "hostile vector")) === sentinel + end + + column = Parquet.StructVector(["x", "y"], + AbstractVector[Int32[1], Int32[2]]) + sentinel = N5HMutableSentinel(72) + preflight = _ -> begin + column.children[2] = N5HThrowMetricChild(sentinel) + return nothing, Int64(0) + end + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + err = n5herror() do + Parquet._nestedwritefields( + Pair{String,AbstractVector}["s" => column], 1, limits, budget; + preflight=preflight) + end + @test err isa ArgumentError + @test err !== sentinel + @test Parquet._budgetused(budget) == 0 + + view = Parquet.StructValue(["x"], + AbstractVector[N5HFalseBoundsChild()], 1) + @test_throws ArgumentError Parquet._validatestructvalue(view) + sentinel = N5HMutableSentinel(73) + view = Parquet.StructValue(["x"], + AbstractVector[N5HThrowBoundsChild(sentinel)], 1) + @test n5herror(() -> Parquet._validatestructvalue(view)) === sentinel + + sentinel = N5HMutableSentinel(74) + outer = N5HShiftOuter(N5HThrowMetricChild(sentinel), false) + budget = Parquet._LiveByteBudget(limits) + err = n5herror() do + Parquet._writefields((x=outer,), limits, budget) + end + @test err isa ArgumentError + @test err !== sentinel + @test outer.shifted + @test Parquet._budgetused(budget) == 0 + + sentinel = N5HMutableSentinel(75) + outer = N5HShiftOuter(N5HThrowMetricChild(sentinel), false) + budget = Parquet._LiveByteBudget(limits) + err = n5herror() do + Parquet._writefields((x=[outer],), limits, budget) + end + @test err isa ArgumentError + @test err !== sentinel + @test outer.shifted + @test Parquet._budgetused(budget) == 0 +end + + +@testset "N5-C hidden package view topology" begin + factories = (n5hhiddenlistattack, n5hhiddenmapattack, + n5hhiddendirectlistattack, n5hhiddendirectmapattack, + n5hhiddendirectlistkeyattack, n5hhiddendirectmapkeyattack) + for factory in factories + table, callback, owner = factory() + err = n5hassertprivatefailure(table) + @test !(err isa BoundsError) + @test callback.calls == 1 + @test owner.offsets[2] == 0 + + table, callback, owner = factory() + n5hpublicatomic(table) + @test callback.calls == 1 + @test owner.offsets[2] == 0 + end + + mktempdir() do directory + path = joinpath(directory, "hidden-view.parquet") + sentinel = UInt8[0xde, 0xad, 0xbe, 0xef] + for factory in factories + Base.write(path, sentinel) + table, callback, owner = factory() + err = n5herror() do + Parquet.write(path, table; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test !(err isa BoundsError) + @test callback.calls == 1 + @test owner.offsets[2] == 0 + @test Base.read(path) == sentinel + end + return + end + + inner = Parquet.ListVector(Int32[0, 1, 2], Int32[10, 20]) + outer = Parquet.ListValue(inner, 1, 2) + bytes = Parquet._encodefile((x=[outer],); checksum=false, + pageindex=false) + reread = Parquet.Table(bytes) + try + observed = [collect(item) for item in reread.columns.x[1]] + @test observed == [Int32[10], Int32[20]] + finally + close(reread) + end + + inner = Parquet.MapVector(Int32[0, 1, 2], Int32[1, 2], + Int32[10, 20]) + outer = Parquet.ListValue(inner, 1, 2) + bytes = Parquet._encodefile((x=[outer],); checksum=false, + pageindex=false) + reread = Parquet.Table(bytes) + try + observed = [collect(item) for item in reread.columns.x[1]] + @test observed == [[Int32(1) => Int32(10)], + [Int32(2) => Int32(20)]] + finally + close(reread) + end + + inner = Parquet.ListVector(Int32[0, 1, 2], Int32[10, 20]) + bytes = Parquet._encodefile((x=[inner],); checksum=false, + pageindex=false) + reread = Parquet.Table(bytes) + try + observed = [collect(item) for item in reread.columns.x[1]] + @test observed == [Int32[10], Int32[20]] + finally + close(reread) + end + + inner = Parquet.MapVector(Int32[0, 1, 2], Int32[1, 2], + Int32[10, 20]) + bytes = Parquet._encodefile((x=[inner],); checksum=false, + pageindex=false) + reread = Parquet.Table(bytes) + try + observed = [collect(item) for item in reread.columns.x[1]] + @test observed == [[Int32(1) => Int32(10)], + [Int32(2) => Int32(20)]] + finally + close(reread) + end + + for (inner, expected) in ( + (Parquet.ListVector(Int32[0, 1, 2], Int32[10, 20]), + [Int32[10], Int32[20]]), + (Parquet.MapVector(Int32[0, 1, 2], Int32[1, 2], + Int32[10, 20]), + [[Int32(1) => Int32(10)], [Int32(2) => Int32(20)]])) + column = Parquet.MapVector(Int32[0, 1], [inner], Int32[7]) + bytes = Parquet._encodefile((m=column,); checksum=false, + pageindex=false) + reread = Parquet.Table(bytes) + try + pair = only(reread.columns.m[1]) + observed = [collect(item) for item in pair.first] + @test observed == expected + @test pair.second == Int32(7) + finally + close(reread) + end + end + + logical = Parquet.LogicalColumn(String["alpha", "beta"], :enum) + fixed = Parquet.FixedByteArrayVector{Vector{UInt8}}( + [UInt8[0x01, 0x02], UInt8[0x03, 0x04]], Int32(2)) + for (inner, expected) in ((logical, ["alpha", "beta"]), + (fixed, [UInt8[0x01, 0x02], UInt8[0x03, 0x04]])) + bytes = Parquet._encodefile((x=[inner],); checksum=false, + pageindex=false) + reread = Parquet.Table(bytes) + try + @test collect(reread.columns.x[1]) == expected + finally + close(reread) + end + end + + backing = N5HShiftOuter("alpha", false) + logical = Parquet.LogicalColumn(backing, :enum) + err = n5hassertprivatefailure((x=[logical],)) + @test err isa ArgumentError + @test backing.shifted + + inner = Parquet.StructVector(["x"], [Int32[1]]) + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields((x=[inner],), limits, budget) + end + @test err isa ArgumentError + @test occursin("StructValue needs its owning StructVector", + sprint(showerror, err)) + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) +end + + +@testset "N5-C empty MAP source trace alignment" begin + values = Parquet.MapVector(Int32[0, 0, 0, 2, 3], + String["a", "b", "c"], + Union{Missing,Int32}[missing, 2, 3]; + validity=Bool[false, true, true, true]) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile((attrs=values,); + pageversion=pageversion, checksum=false, pageindex=false) + reread = Parquet.Table(bytes) + try + @test reread.columns.attrs[1] === missing + @test isempty(reread.columns.attrs[2]) + @test isequal(collect(reread.columns.attrs[3]), + Pair{String,Union{Missing,Int32}}[ + "a" => missing, "b" => Int32(2)]) + @test collect(reread.columns.attrs[4]) == + ["c" => Int32(3)] + finally + close(reread) + end + end +end + + +@testset "N5-C nested row witnesses" begin + for trigger in (1, 2, 3) + table, callback = n5hnestedlistmutation(trigger) + err = n5hassertprivatefailure(table) + @test err isa ArgumentError + @test !(err isa BoundsError) + @test callback.calls == trigger + end + for trigger in (1, 2) + table, callback = n5hprovenancenestedlist(trigger) + try + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields(table, limits, budget) + end + @test err isa ArgumentError + @test !(err isa BoundsError) + @test callback.calls == trigger + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + finally + close(table) + end + end +end + + +@testset "N5-C provenance one-read MAP-key authority" begin + sequence = Int32[1, 2, 1, 1, 3, 1] + table, keys = n5hprovenancekeysequence(sequence) + try + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + err = n5herror() do + Parquet._writefields(table, limits, budget) + end + @test err isa ArgumentError + @test !(err isa BoundsError) + @test keys.calls == 2 + @test Parquet._budgetused(budget) == 64 + Parquet._release!(budget, 64) + finally + close(table) + end + + table, keys = n5hprovenancekeysequence(sequence) + try + sink = IOBuffer() + Base.write(sink, UInt8[0xa5, 0x5a]) + err = n5herror() do + Parquet.write(sink, table; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test keys.calls == 2 + @test take!(sink) == UInt8[0xa5, 0x5a] + finally + close(table) + end + + for factory in (n5hprovenancemapphase, n5hprovenancemapaxis) + for trigger in (3, 4) + table, callback = factory(trigger) + try + bytes = Parquet._encodefile(table; checksum=false, + pageindex=false) + @test callback.calls == 2 + reread = Parquet.Table(bytes) + try + @test reread.rows == 1 + @test length(reread.columns.m) == 1 + @test only(reread.columns.m)[1] == + (Int32(1) => Int32(2)) + finally + close(reread) + end + finally + close(table) + end + end + end +end + + +@testset "N5-C provenance binding-kind precedence" begin + recursive = Any[ + Int32(1) => Int32(2), + (Int32(1),), + (x=Int32(1),), + Parquet.ListValue(Int32[], 1, 0), + Parquet.MapValue{Int32,Int32,true}(Int32[], Int32[], 1, 0), + Dict(Int32(1) => Int32(2)), + Parquet.StructValue(["x"], AbstractVector[Int32[1]], 1), + ] + for value in recursive + table = n5hprovenancedeclaredkeyattack(value) + try + sink = IOBuffer() + Base.write(sink, UInt8[0xa5, 0x5a]) + err = n5herror() do + Parquet.write(sink, table; checksum=false, pageindex=false) + end + @test err isa ArgumentError + @test !(err isa Union{MethodError,StackOverflowError}) + @test table.columns.m.keys.calls == 1 + @test take!(sink) == UInt8[0xa5, 0x5a] + finally + close(table) + end + end +end diff --git a/test/conformance/n5/julia-fixtures.files.sha256 b/test/conformance/n5/julia-fixtures.files.sha256 new file mode 100644 index 0000000..9ceef60 --- /dev/null +++ b/test/conformance/n5/julia-fixtures.files.sha256 @@ -0,0 +1 @@ +feb1d2ca2dafa0604d0342995556b21beff67bb099dd9a979524778cfb52b5ee julia-fixtures/files.sha256 diff --git a/test/conformance/n5/julia-fixtures/files.sha256 b/test/conformance/n5/julia-fixtures/files.sha256 new file mode 100644 index 0000000..6270eed --- /dev/null +++ b/test/conformance/n5/julia-fixtures/files.sha256 @@ -0,0 +1,353 @@ +e7bdcc002e779d4552f78936929d56f904992a5f3e31704c2090fa701037528d fixture-manifest.tsv +3cd92d1a3e45c93321933efe31b62d29fa7278c134497da08597baa765f129c4 julia/model/direct-list-of-map.v1.parquet +55eb313494bcc4b697ef0af13fc1a6522afbff67f8f42d4931aa86f09987fb08 julia/model/direct-list-of-map.v2.parquet +1501150d96e2efa9114e74b1f0554ac038674f92999babffe25f0699e2720bc1 julia/model/list-rule-1.v1.parquet +320a07c8ad7cb61837e40d1c2c8f79c1201cb826ddda5a1a01b8d020aa2c19e0 julia/model/list-rule-1.v2.parquet +7f086c31f18f378e53a68e6baf9a4c987ff77c2ffcfc406d9e1e686d79ee168b julia/model/list-rule-2.v1.parquet +8de4b568bb8702f35ea07ef5bf17a0c3f0a61481d7bc8e26e8089bd30f9e8e5e julia/model/list-rule-2.v2.parquet +b48b255f20ef9b42ee43d750e74c3c443029cc710fd3779702491519c2365789 julia/model/list-rule-3.v1.parquet +7976722c0954843510ff9b63e0cd49b5f2893a9e7589b776b4654143d46301b9 julia/model/list-rule-3.v2.parquet +987ebdaa8f335dfde3e5d39f16e335642d801335a97cd77add9832cb0dd0230b julia/model/list-rule-4-array.v1.parquet +67864226a1f3cea4c9fec7ca55a6d77ed0a7ed159e35fc106756e3217a84ea01 julia/model/list-rule-4-array.v2.parquet +0c996d1f5fefd0bac9a2288b536e97d98def99f24aa69122e4f7869ea0fd1510 julia/model/list-rule-4-tuple.v1.parquet +2f6ac7955f51ac3cb23d88ffddb41d7dfca3a99fc91e42ccbb0c6ed531c28f0c julia/model/list-rule-4-tuple.v2.parquet +06b0b4672e66b30ccd8f2bc9d2d19d8aa61c897b7295ab4e6f2626c7c7fe9b88 julia/model/list-rule-5-extended.v1.parquet +da7da6bdc76cb788e7ab6bcf53d8c3fdb6e5b72ddf52d3b2db1b0579488a1e65 julia/model/list-rule-5-extended.v2.parquet +b3891b7e0f0f78e1259a10eee6c60e7b37339b9b1a40da91c58ec231ef578f1e julia/model/list-rule-5-paired.v1.parquet +3c771ccaab1e33f342ee99aa05b68c556722b1fbda86dea87f11de9d842feeeb julia/model/list-rule-5-paired.v2.parquet +2ef6f689de0f5590e6c2bdbc42ce49682aa69ffdbe62903adb4c0c83d1600641 julia/model/list-rule-5-required.v1.parquet +099aef2c8d8de2d4abae1b225fd67e0d7f9005a50d262d6fb61a7f9253e0002d julia/model/list-rule-5-required.v2.parquet +beb097ba4292c9b54928442895edbd709e342146fff39e57564ba1f7292d8f23 julia/model/map-key-only.v1.parquet +cd7eea07de636d91e381e263c230652e68c0ab06b726e00edea1f40491dc246c julia/model/map-key-only.v2.parquet +a695dca0ca686f153b0fe3636f0baa1eff8a209df80f69a8d116a77d1c5e82cf julia/model/map-optional-key.v1.parquet +f866646481c7eccca7948d801c0e0fdc0f67447cce2604ee28c0a5e96f513501 julia/model/map-optional-key.v2.parquet +1780198fdd9764e85e17ec2a78a56cc27940657087a5f81cf7c15321d6ac74cc julia/model/map-standard.v1.parquet +356c5e36c7e69b1074a48e2a920255accf2cf0f6d78a6bad46dc6b61cd630e07 julia/model/map-standard.v2.parquet +989a46daaeefedba745c885541d9e4d0dd703360f49b98736d6f27efaca45257 julia/model/schema-provenance.v1.parquet +0b95c7dcc128d829c9b07c0a7db15adc8de720da8b40516a798aab46f10305b3 julia/model/schema-provenance.v2.parquet +4c52263c0bc6f3eb7598aaf8f0c2038531bc7f847f0e8a45bad77cce68f4ed35 julia/property/generated-0002-v1-brotli.parquet +bd3d065e0314c97806cc59c84ec4740a49c0a960fbc01d10b487f44be333e9bd julia/property/generated-0002-v1-gzip.parquet +6a8af5309e1185c00615773ec5c578dbc2407700ce52659a74affb41a2edde05 julia/property/generated-0002-v1-lz4_raw.parquet +e0b67b9145f0dd5d223a540e40237772891e40c692131a48df3f769a9fe42822 julia/property/generated-0002-v1-snappy.parquet +d95a08cf23ec4ebe971b4874de5460a71b43baed2ddcae900ee13aefbd8df6d1 julia/property/generated-0002-v1-uncompressed.parquet +dadebc22273aa2dc47b12fa4a4386563135d82b54cfaac7ef31af3653a7e7fd7 julia/property/generated-0002-v1-zstd.parquet +470f2a6d2f5c5f6949def9205e7146ac8acd5249b343065447098250ecce605a julia/property/generated-0004-v1-brotli.parquet +ca2d2026b4947a2daa5f38b713b70824107e8d03b900899209ff00c3c0e87bce julia/property/generated-0004-v1-gzip.parquet +b6e96c0d2140294f40e44efbd7d93aecae1b3d56bba0c022f0b937902425d6ca julia/property/generated-0004-v1-lz4_raw.parquet +9604c353851f32cfa15e7c579b5e560ff0dbefa3958ee348f1e7431bbaaa310f julia/property/generated-0004-v1-snappy.parquet +6b7aa5127af8b21e107e42cfdc6324152965ed4637ac939866216da981e695f6 julia/property/generated-0004-v1-uncompressed.parquet +fb80f46aa51f7b5b1ee8d2e935f4329bc130306067e0d1d572dc618b51eeb9dd julia/property/generated-0004-v1-zstd.parquet +d580a983d9080ffe1a4f1c639c07f95b72f4a78a7fd8943b31388f3db790d752 julia/property/generated-0005-v2-brotli.parquet +6cd0c6381540b6f81ac6fd041505c712f003fd476b664ba5ec61f4db35d77957 julia/property/generated-0005-v2-gzip.parquet +b3271eb65b2c955448628399e55480994aa838f0ff011d7e09498c4c10cf4835 julia/property/generated-0005-v2-lz4_raw.parquet +0ac610ab2f5dcfefc7290f4e7f045cfcf398e9e7de80554e5c075a45a9f148d7 julia/property/generated-0005-v2-snappy.parquet +90224b72fc575a9fe620b93b11760e187ff7cf117a5641a08fdfb111444a34b7 julia/property/generated-0005-v2-uncompressed.parquet +e1b01ac66e24fb286038cb74449d57717443482cffd5add212f7209187b61190 julia/property/generated-0005-v2-zstd.parquet +3229de621e58063dc8e54e7917bccdbdb2f5f7862f67389661d54adebf1a1f4f julia/property/generated-0006-v1-brotli.parquet +5e2a7e6c915bbbdbbfaaab840898de90c9677c4fdd25cc9b08aaf1397ef50876 julia/property/generated-0006-v1-gzip.parquet +1d2a6e0763caa72fde602cf3a5158fb9fe1b77a487723780135b26b5858181be julia/property/generated-0006-v1-lz4_raw.parquet +0cdd2e50493ff1786fd07e69d6e039e3acf22cac39413b40f535d6fad0df681d julia/property/generated-0006-v1-snappy.parquet +f0a8b7e3ef263d562ebb4ebdb778628654e7632dc3a6ba00fdb2dff4e93a3d4d julia/property/generated-0006-v1-uncompressed.parquet +af2509a05c749f4c3b46e3815c51f6196fb45baea4cb232866af38fa4fcd623a julia/property/generated-0006-v1-zstd.parquet +ecd60078edb9b7d8c742d95eb584387ead98d348905588d77049ee2ddb3c63e4 julia/property/generated-0007-v2-brotli.parquet +9941876a4d0ed9ad6325662d5065a91b4fed9f75f482ab5a0f10d4243401c515 julia/property/generated-0007-v2-gzip.parquet +8db9aa30605d421f480e5b6e18cfab03054385fd59f013c10fb39e0716a76384 julia/property/generated-0007-v2-lz4_raw.parquet +ce35740658a531d6fa65cf36286036e75454d0f3ab7c0fd48db3da2668b2e4fd julia/property/generated-0007-v2-snappy.parquet +73e0b2f3aeca0fd57416ee6e1f1242baa3752ae267c79089e3800dd61044e706 julia/property/generated-0007-v2-uncompressed.parquet +aa271051d84a3fefd09655260c488d41858c908d2112de6e54f93ac32eda2036 julia/property/generated-0007-v2-zstd.parquet +6ea0ad9396016feed334f6faac37925c2eb63872bbb3a56c45bd6464633b3cc4 julia/property/generated-0008-v1-brotli.parquet +e38c60bd13289a9480aaec1edf879c0d1925c09ae3e1bfec37cae1faf35f85f9 julia/property/generated-0008-v1-gzip.parquet +969ccaa5870c17908abd06181f71e7786a105b02663f1d57eea8e1fe5c99e09f julia/property/generated-0008-v1-lz4_raw.parquet +2d401978d38b2c0dd63b1ad1b9872854bac561414981cf0dc30c7cf3ff7991ee julia/property/generated-0008-v1-snappy.parquet +5174cba3dd2096381491fd47bf6f9a497f2640a05f803a95eb1f0bd29c48598d julia/property/generated-0008-v1-uncompressed.parquet +c0203aeef6ae8d7e6e88f9319e552bcbec13f79809cf79fb658171eff3a3812e julia/property/generated-0008-v1-zstd.parquet +62f0915ca7411dc35e906d0e41fab32e09ba70eadf7bdb122760609a4c533b26 julia/property/generated-0009-v2-brotli.parquet +b483b5cedfaf16edf67201a9d7cf13ff20b87fdb119f72b33ab4cda7ee0d3c3f julia/property/generated-0009-v2-gzip.parquet +df387db6ef217af1b9025b21262965a608d16a5fa02296f52630f52ad06b4654 julia/property/generated-0009-v2-lz4_raw.parquet +906d43803d0c4939cc5e3477cc997fadf8b25f1a9c061bda7820ae30ef88f9d4 julia/property/generated-0009-v2-snappy.parquet +e26b53294981c361d1cd202c287a7dab38f3134e93a72dd64c0bba785609353d julia/property/generated-0009-v2-uncompressed.parquet +08aba081ff45baa57fb4b3a5f0c3309d8902e8f268acf70737587941c0fce73f julia/property/generated-0009-v2-zstd.parquet +86177f80c869a423c3463e4f4d79fd7ef1c4c20afa11ba4fa58685a938209395 julia/property/generated-0010-v1-brotli.parquet +c2e77b5d267bf2d15dbf936577706c4a66d6852fb9b147105cee95cfbb430492 julia/property/generated-0010-v1-gzip.parquet +54f15384ca731da3f6c143d7bf9fabbc6b08f2d9a61671391caf7479f9a4c37b julia/property/generated-0010-v1-lz4_raw.parquet +3926fb9bb0c833f3a1ea3240c8016a96b82e4e423701df30a77d5de70bca65df julia/property/generated-0010-v1-snappy.parquet +fd0206f9f2599ddf38a8fb91ed518996a56155f172823b385ed2b66a3c191be4 julia/property/generated-0010-v1-uncompressed.parquet +6889010559e0b875e7d4208de19afaeb60a9adad781c134f5ef716c668138fb0 julia/property/generated-0010-v1-zstd.parquet +329e5c69bf4cf4e59ca2d87fb93d1068afbe14a55db743c7fd3ea9ee6d2b1dc8 julia/property/generated-0011-v2-brotli.parquet +ba241f6ec00c55a98864886bc32c5494931fb94b999f31ef7051d2d63a96d1b3 julia/property/generated-0011-v2-gzip.parquet +e9f3ea6f819b22b37661a398fde0f16610750ae8d38e187dfbc134f1cddbd158 julia/property/generated-0011-v2-lz4_raw.parquet +090ea05ae46f5dd45b3d858e593af5a5c73785319378f1b207aa6d2680ada1db julia/property/generated-0011-v2-snappy.parquet +ff6992972160170aff728ff3f7c53473c4190e64bc53e315128194799969fca8 julia/property/generated-0011-v2-uncompressed.parquet +0196152849dd81597908abec7588bceea7b6f3d4e3eb6e18e427fbec846aa15a julia/property/generated-0011-v2-zstd.parquet +1bd5c130fe288087daf514e8acfff9b24e4dc0d8a41af903b97dc7e9551cb2fd julia/property/generated-0014-v1-brotli.parquet +d46975dbaa3bd7f5c9335e2a7afecc1c51018afcf49973d5fa7436630ae0ee73 julia/property/generated-0014-v1-gzip.parquet +55cf0a1b8c936ae0b5e0409b8cb6e3501404a5821dcf473906787d27836ca669 julia/property/generated-0014-v1-lz4_raw.parquet +7b0a73d0df98bb54e2071b920eebd00053871c2f273c98d4970114de0c53984e julia/property/generated-0014-v1-snappy.parquet +01bba38899a61aca7329887710a9d073972fe8f2de01ae522b33d7932bb79f63 julia/property/generated-0014-v1-uncompressed.parquet +a522c5284b3a6593a47f30b9cfc1402c66f5b661164b6dbe81c277ee0a86cc7a julia/property/generated-0014-v1-zstd.parquet +cff5c741b35a9cdfda79e4fb53df6ccad367d30ac4a59d87714c7aabf12d756f julia/property/generated-0015-v2-brotli.parquet +94b22cb22ac553d25006924d51ec98a403884eed256a0d45435334f664781f7d julia/property/generated-0015-v2-gzip.parquet +028acbdbc55150c27b2a90db12b38cd0054c39c50c97d086bea222956b0f0169 julia/property/generated-0015-v2-lz4_raw.parquet +66d06e79d863df5c4137a976470ea8c98df428637f90a5796a90147712b12f66 julia/property/generated-0015-v2-snappy.parquet +c83e5973b6e3c1893b27731b96605ee2a947f4c4790a78be5b5dabb58230e3a5 julia/property/generated-0015-v2-uncompressed.parquet +e9afe072d5495ec59ecb64804bab60a03e4f28bd68c2d6813b8800c952e7ba6a julia/property/generated-0015-v2-zstd.parquet +4a1049e083efd4db62d1950ae55b60a02b1f11dd48926ae3ec6f9f5f7247b3a5 julia/property/generated-0018-v1-brotli.parquet +ec54eeb5bd624d40de14a0648696b1e142fcb56cba73f9c0a27815cde72dd62d julia/property/generated-0018-v1-gzip.parquet +e652771c3b56087db00ecb99a875d4b0d967be6f5b25c9d5634db3cec48eba9b julia/property/generated-0018-v1-lz4_raw.parquet +5c2d52f8fb338c2d252a1156caeea089c8b3ae8a2e7aad410cb3224a8e2c31d3 julia/property/generated-0018-v1-snappy.parquet +ac5e8e0371fd2f499f09df1aadea4648eab0f5aa3594729baceccb2360c404cd julia/property/generated-0018-v1-uncompressed.parquet +85381544858a2293267cf513d35934d16c5500c4b9c3ff29076097f410e970d7 julia/property/generated-0018-v1-zstd.parquet +129a0c7e6562926f4ab35506e26772f4a85bc6e1abcbbc418a52b9f759011007 julia/property/generated-0028-v1-brotli.parquet +76630e94386c56c1fe3cb3838bcefbb8f07a2721851cb370e45f493342adae76 julia/property/generated-0028-v1-gzip.parquet +7076ff328050bbd7233ad65dc96ed7180e5a9b1caee2197f63e278d73b555149 julia/property/generated-0028-v1-lz4_raw.parquet +d66e711a68fc4bbe8e393dcb42ea2740fa01c3ac69df62fb02281c43a3b7c8f6 julia/property/generated-0028-v1-snappy.parquet +0f79f24eb9ac8c4a3972a18746c620ea574da27b48f661ce1396a8869afeade6 julia/property/generated-0028-v1-uncompressed.parquet +619c6b8655ec07a70ed2187c43e99465394222a7a8acf03467215e4a55c82c56 julia/property/generated-0028-v1-zstd.parquet +6edda09d48570e719691edf0227bcf05a5fda8da6a7eddbcc90ca0e5303713a9 julia/property/generated-0034-v2-brotli.parquet +62353c41290f30a739ebb19e3a62020990c537b1beb43bde1e627bbb60e84845 julia/property/generated-0034-v2-gzip.parquet +53257f2e29a58c1f4ac948597f8adeca32de74457b141a69220903494bbdb3ea julia/property/generated-0034-v2-lz4_raw.parquet +e22ebbae91d2795a7ddc66eb430155df7c6ce24dfd530f4960ce93e2d6137aca julia/property/generated-0034-v2-snappy.parquet +0da351162998efffaef53c035af079af72b952acea593b8f3554702b16a2e9e8 julia/property/generated-0034-v2-uncompressed.parquet +380cb85adb83b197eeb91c03a16c43c065e43ec04da394bb6822733735fba20e julia/property/generated-0034-v2-zstd.parquet +5ad7fbf6fdb9235427ee68d580b8e324d7827a70f058d291309962cb8785c0c0 julia/property/generated-0038-v2-brotli.parquet +027f0cc87a4b5de6ecd96ecf8231d55f3b208b5bbf0468a53680ae540b0c7644 julia/property/generated-0038-v2-gzip.parquet +c9f2d02e46cb65c9a1fd891a041f792e16139d6e8d69d3262ba8bffc932b7b03 julia/property/generated-0038-v2-lz4_raw.parquet +e88612da352e14bf9068e0d3df599699ef431053685ccc249649d5c64ca96c08 julia/property/generated-0038-v2-snappy.parquet +4e6abb4e0d6ceb60815129f1544bc0209a9a0f3d35bd99de647f73a299d108a9 julia/property/generated-0038-v2-uncompressed.parquet +bfb7d9529dfdcf3487fc460f97b6386920c728ecf30f41ff771cff5b0fe595bd julia/property/generated-0038-v2-zstd.parquet +d4f57e1b5fc618c74813e6a90a21a21a8be246c1625364502825a90f58654d2c julia/property/generated-0048-v2-brotli.parquet +95ac41365a6f5f49278443f03bb80b82a07431f426c1277f7d1b7cd20734430b julia/property/generated-0048-v2-gzip.parquet +2ae57bf5188303172e1cc7e0155f2aa65aeed4c300a46bf0f6b7a85296922f1c julia/property/generated-0048-v2-lz4_raw.parquet +c2e56e1a6b10e43ff4f14c8f4a5fd3982a16c7a7b88652ab27ff3332f030e7a5 julia/property/generated-0048-v2-snappy.parquet +9461f475bd285c22adc2c1a74c8d905b2022fd08a381d83623921e73091135e4 julia/property/generated-0048-v2-uncompressed.parquet +c9b5bee8cfc84d0cc8d7ca9b2bdbbfc52b4628ebbd1f4ad7b88c76639d405c40 julia/property/generated-0048-v2-zstd.parquet +de4a8f6f02252f661c64b0c71d8e74987a6af57872cadf68d807841b47f27590 julia/property/generated-0058-v2-brotli.parquet +e1f65729ca9da5813586ad862f43e96ab4fcddecf65b9a16858961038b7c809c julia/property/generated-0058-v2-gzip.parquet +41be1bdad62789707cc1a135a1893163e3f3bbbfcdb80f7b6c5e9487f1f53a65 julia/property/generated-0058-v2-lz4_raw.parquet +b9ad0e2bf83e7ff1badf9b7458ebb001a016f0b6bfc3a29a3f3161d91299ede7 julia/property/generated-0058-v2-snappy.parquet +d43a2eb1090974bc470e7eea2a76841c821306ec995706b384725f51e9de5ea6 julia/property/generated-0058-v2-uncompressed.parquet +f2631a99b551973aa88dfaff2f2b54f18e7e43092b57865fb3e0ff656511e015 julia/property/generated-0058-v2-zstd.parquet +842b3b2d51d4d88f7231492971fe4f427010f3fb569970ee726021800e4aa5ee julia/property/generated-0062-v2-brotli.parquet +375161c057355f0237583dd1d60bca7d26da1b95e63a9aa6dd999085f4ff8323 julia/property/generated-0062-v2-gzip.parquet +035d23477076d704a8cfe3dc93ea8b2e89e36657bdb167dd39085578fb1c48d7 julia/property/generated-0062-v2-lz4_raw.parquet +4652e73712b2907568922910ea4557886c250f3061a441da6e3207944a253e26 julia/property/generated-0062-v2-snappy.parquet +a6a1d3edc66ea53e88e67aac480da9f6e7417bea1ca423f365706fd537320c84 julia/property/generated-0062-v2-uncompressed.parquet +b72fb3e6d50dea4a9fa3ac38b4ece6b03c6088389c14d49fc18fa0c20bb3d457 julia/property/generated-0062-v2-zstd.parquet +5625bc4932c5cbbbdac746275fb390418edc70998d8ef8fa92ebddcfafaac59f julia/property/generated-0096-v2-brotli.parquet +c30b9786dad53e7fb9cef306ae2a37ac72b4e380a9bb69f3e2a677a365cee72b julia/property/generated-0096-v2-gzip.parquet +95ef9e11914c472e812f74e910ca71923de940239aa3885316b17823b707ccc1 julia/property/generated-0096-v2-lz4_raw.parquet +a8ab83122115d1b460c15153d3da506b5152e5c2bb16a381f90c8678b2fc5855 julia/property/generated-0096-v2-snappy.parquet +4c9510f3fa4086d93a9974d5f53d167420e5da84fd55a305af1c41bf7832a3bd julia/property/generated-0096-v2-uncompressed.parquet +6c373d2d0375604bf66bc5bfeabe66d27b671140ac2977b6aa526aaa9cd02775 julia/property/generated-0096-v2-zstd.parquet +9ea9011689b8e0ec83c60939017c36ce215b973e4e8b995115018e92704d5c45 julia/property/generated-0106-v2-brotli.parquet +c1af4a2b192f23d2cfe6cbe5c966c729c703c300c7462c40cee244324cbf91b3 julia/property/generated-0106-v2-gzip.parquet +d31b395d03a6597db1eb57b5c76502e69fcaf4c058d4e33cd6f239ddec6d9264 julia/property/generated-0106-v2-lz4_raw.parquet +1363b2afc42817100c03a59aff5214fd8abd239c959a8a364c1267b71403501b julia/property/generated-0106-v2-snappy.parquet +23927d0148d0471cb403ef071db11942e861f687676eb547f3ed85672e01e36d julia/property/generated-0106-v2-uncompressed.parquet +8b45738b2ad1723013575894a3fb854e4747bdf0848fda3a5e26464e62aa4dfb julia/property/generated-0106-v2-zstd.parquet +8b7680549d83e5aaf5fd06529bc11c895c8912bed07f5d768b8a0167a07c666b julia/property/generated-0107-v1-brotli.parquet +5a971be9aff80231b3ca44afbc53b14f50066ac6564f81976ac52490391e9327 julia/property/generated-0107-v1-gzip.parquet +1808e7743750f387737cd11b75dd40afc0555868b82e69dcbb6afa3c9fb7afb8 julia/property/generated-0107-v1-lz4_raw.parquet +f78c20178f34150fc9b7d0640a3b4fea8938f37520896f401daa93e3378bfe0e julia/property/generated-0107-v1-snappy.parquet +8e739a9d444efd3b8f86936461574533f8712ccc62f558efbaea95a88772603a julia/property/generated-0107-v1-uncompressed.parquet +dd7666c4c882965d925acbdd6d0b8aec1f55fa50bca3ca7d8a264659cbed466b julia/property/generated-0107-v1-zstd.parquet +aa8ced8071e1f522a15c99a824a5b4bb936df197b9a4becb206cdf829ae0f2db julia/property/generated-0117-v1-brotli.parquet +d70bbb449a0a2c69ddb2f61b97cff41df2bc5a80712bcde8d61de72a111d9dab julia/property/generated-0117-v1-gzip.parquet +07f6401ff94b99450b66f60561110aa25763f9340efafc5748da2e72ce50c76f julia/property/generated-0117-v1-lz4_raw.parquet +f43ce5013aea4784cc23f0a39fa17df2e5fc2c505bb4cf9e37d7c1011931667f julia/property/generated-0117-v1-snappy.parquet +74d10b8ca2124925be4e436462d0dc04e41c45c65a332aa04763a746f33e2fac julia/property/generated-0117-v1-uncompressed.parquet +de62e5733770ea09877dab879eff8b3d477d97fc9523174c167b46e81ba15722 julia/property/generated-0117-v1-zstd.parquet +242307ce4a7d3571df8ad625904e0138c608a325f3f20c71cce953baa265c6fd julia/property/generated-0122-v2-brotli.parquet +4f9221c2c25e40ef0ec4824a181dac9df68b9e5acd5045d3ee3f3df3e9075c34 julia/property/generated-0122-v2-gzip.parquet +559532d6a23c0c65a3a7494c071e24a49321adda8edd51cbc12adcd06dfd46af julia/property/generated-0122-v2-lz4_raw.parquet +fe43353a178efa494fdb6b9985383a91bcbac6f97111e8b9f3c7df355b549068 julia/property/generated-0122-v2-snappy.parquet +d083e87b924925b8ad7407841ace216b86abe58d3720cbbe46e93afc6b0763f1 julia/property/generated-0122-v2-uncompressed.parquet +ae69b09dcdca2334d1f672843b749a289186c268db7287e81e88354694add9df julia/property/generated-0122-v2-zstd.parquet +41a87713e90bdfdfb76a01d28e63f3bb478325b38d22b693182f627119eb33cf julia/property/generated-0123-v1-brotli.parquet +c68589c51eee0e3a32794157242399a67795a131e15b3b8cecc78c25b46250c5 julia/property/generated-0123-v1-gzip.parquet +e1608b8489b6da60abaece8104fbb9ca489e8a5fc7f11d2157aba706c95d993f julia/property/generated-0123-v1-lz4_raw.parquet +79201ccbb8345be91f651c119a77b5f481775144d85784e938532d9d9d01e614 julia/property/generated-0123-v1-snappy.parquet +239a8fad2fc0d36640a9948d15b8fbff9ba0902540bc9437d9a78602f731080c julia/property/generated-0123-v1-uncompressed.parquet +65ba1030bafaeb9f7a603e877ff27e999d28da79728bf94ba124fb4f64bb7f31 julia/property/generated-0123-v1-zstd.parquet +b72ca7250bbeec02a714bf3c8b5c13e2dbf30c66264905957314d7a13ba18d16 julia/property/generated-0144-v2-brotli.parquet +62b7e66658cfaa2993474ff4b4331fa897f7879d54817a30ec1f749b894229dc julia/property/generated-0144-v2-gzip.parquet +efcaf5caedb62083bc5560af6942dc18271303eb2e9954c7e2b62343d5baf866 julia/property/generated-0144-v2-lz4_raw.parquet +f0c068d591f433e8503a111ff75a28889253d5101a679ef31550c6b9cb4e74a4 julia/property/generated-0144-v2-snappy.parquet +9184b278ecbde6ce8d6adfd4a3a6cafffb836f042ebfacba1a5a560435a6a8e8 julia/property/generated-0144-v2-uncompressed.parquet +282e2c701498e2958254fc947308d289af2d52b413690ccb3fe35b3292af20ca julia/property/generated-0144-v2-zstd.parquet +3b8cf3a99f866819fccaac561f70d11e2c0957a21ea761dfd4212b6e3ec5a74a julia/property/generated-0145-v1-brotli.parquet +a0804e8c338a8995f7ba041ce4bac0b71b785cfdf8082b4e035514ddf2273a11 julia/property/generated-0145-v1-gzip.parquet +b12a7976be84f15d9fe1d03da40d95373719438d4c8c32deb9766ab70ec68f5d julia/property/generated-0145-v1-lz4_raw.parquet +4b952c788b01bbd257f544c92a7a6639bdb88e4634c4422f1aad66ff57a1718c julia/property/generated-0145-v1-snappy.parquet +2a62d573ae8c6a385e5d89e8437c6e5b0c4cc8c1c50e581b5ee2b764a7e6b94a julia/property/generated-0145-v1-uncompressed.parquet +c2ee4c58f5a4ca9ad53f6e59b2b0c3bfdc4b6a99d845e8cc211f71e4cab4aa3e julia/property/generated-0145-v1-zstd.parquet +3dd43312ed90e625120600d9f6cec2b8296c7c430208aff393083a1cc5897446 julia/property/generated-0148-v2-brotli.parquet +2417e12ceb6b45d359eb81d6ee1b43586f6bfb52c8f8ffb28485a245e139cc46 julia/property/generated-0148-v2-gzip.parquet +fce64d41be8e6b8851743eed8b12a1236d198fd85b32054195ebdbcfc97eb463 julia/property/generated-0148-v2-lz4_raw.parquet +e9090565416005e2525656591e5fb4546a9705d854d5a5f11173f083f3f5417f julia/property/generated-0148-v2-snappy.parquet +18849f8a7f133c8b3c4916a9afcdd13f3a1b1a83bc275d47d0fa7efdd23da3f2 julia/property/generated-0148-v2-uncompressed.parquet +d8cc54dc2a8d467221e227063b4232d87fcf036942f8829e90ca7214a087f64c julia/property/generated-0148-v2-zstd.parquet +3482317e88afe245a159e8dba3468e614b238097a81fb7d21bc17a825b8860da julia/property/generated-0178-v1-brotli.parquet +51af92f6754c830aa89ae00d730f09b712535eedd7787f5d323a35e843365fb9 julia/property/generated-0178-v1-gzip.parquet +845b45a555d839b647615c683f7b021cba6e302803331566aa5254b939dfd87c julia/property/generated-0178-v1-lz4_raw.parquet +4940ffda40a210679fe6000ff2c19c91b548ac1a5c3a5ef203a12583384974d4 julia/property/generated-0178-v1-snappy.parquet +897baaed11dde7e8c9172a666fd9acdce838bf8c16e54a24a5b9786ed0d8524e julia/property/generated-0178-v1-uncompressed.parquet +cc39cd58f6d4e122f1f1332671566a45dfe62f0eb9519880995afe6618122901 julia/property/generated-0178-v1-zstd.parquet +d09dd44d135b8eb348a3ff54f0257fa6dbc0648c08b8ffa09e76c03f06b54086 julia/property/generated-0191-v2-brotli.parquet +dfcb1e27fb957ca45329ff661b5baa92200d18c0948935e3bffc67f81518e320 julia/property/generated-0191-v2-gzip.parquet +2afe65db767440a3b8ccb85701425b91998cbfef451ece24faf35f5b3ca0bb99 julia/property/generated-0191-v2-lz4_raw.parquet +6d3940784ce7ac1fa1bdef55efa6da1a6613024bdc94427759b13ef4813de528 julia/property/generated-0191-v2-snappy.parquet +914d6869b729367dab753d8caaecd2171c9d466d317bb1edc052da970e54834c julia/property/generated-0191-v2-uncompressed.parquet +7f0a930b1365398fb595551eea265238b9b4d320011d7f6e392958056d09b077 julia/property/generated-0191-v2-zstd.parquet +3025ed8d78da6ad0638f99e535ffdb66b5fa73dcb3e3e32762042ed1f4f8085f julia/property/generated-0202-v2-brotli.parquet +7f97caf5318185ca59cd1d11ab0b8ad6c18548b1630de151402885f9c3c0aaa2 julia/property/generated-0202-v2-gzip.parquet +a601f114bd71eaa371cb47134063c7c498c804de00326b8aaa0edd215c34d768 julia/property/generated-0202-v2-lz4_raw.parquet +aadaaa13a90872df62258193e9dbdc211c1bd1f8795517c056823cc9d7fac7ee julia/property/generated-0202-v2-snappy.parquet +3d37eac2ddb0d4cf3dfaa30d1d3d2b93927073886a387c3a67d00eecc48ac76b julia/property/generated-0202-v2-uncompressed.parquet +bf4a0f6ee136b1a6b706873ee8c2f6c77758be4e8f5bb630f20e19342167137c julia/property/generated-0202-v2-zstd.parquet +f37a9461c080ae22bb2772096f375b15579af426079e78a1ad9d914a3efda186 julia/property/generated-0249-v1-brotli.parquet +f85151062445dc7a43bd4d873c582e5a5a40fa0fbc3870193ee0ef215249b474 julia/property/generated-0249-v1-gzip.parquet +a1590c1b3dda367851b585e3c470ce2447635a38527cd483015654e8240b6434 julia/property/generated-0249-v1-lz4_raw.parquet +9a779936dc5e17570b030d67020b07e35325b01a401ca07f9ba80a16320932a6 julia/property/generated-0249-v1-snappy.parquet +6dbac686a7041c3bcf82b9fef229557ef720ea1e5c23a6cd9e919e81a0e609d8 julia/property/generated-0249-v1-uncompressed.parquet +13ed8aacb78ffdfb57289ad4c860b87fea66922845f093d9570a35d604ca4bed julia/property/generated-0249-v1-zstd.parquet +e5b7455926b110110f87ba49321a2553f210f79ca76f4bee1aa653b2f62e061c julia/property/generated-0251-v1-brotli.parquet +84062ae72e4c44bdf10b7c45b9d016df6660ff55c87f1a6bf9305cd4687fa850 julia/property/generated-0251-v1-gzip.parquet +59e0ba63b19a2b1f5f6659342a7d2b5fcace31d3064f8ab86261d6459a6ac8a4 julia/property/generated-0251-v1-lz4_raw.parquet +d27e85b23d4c4c8477584a33d2ef805dff5a034769f1a686aae1b2cc91b9fe1f julia/property/generated-0251-v1-snappy.parquet +aca5729d1c9431776b01d2d948b0854deffa6dee6c6784a74acb885f1e85d6cc julia/property/generated-0251-v1-uncompressed.parquet +a5935c6e05ff0697aed875b7bf6db089ae1b966f1a66e2e4d3e4d758b8429965 julia/property/generated-0251-v1-zstd.parquet +6d0b7ea1d1c066a8226ec78d7cfbf47232cbce5f0895267fd6033671ae620e82 julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v1.parquet +aa1bc9a936c93d5fa8db4267d0cd45fb8c564c892b362aff120651d479e1e56a julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v2.parquet +5fbad40b085b1e68911907b498f07f580abc70fa043adc342de939b6d1441485 julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet +81a6be40524b8efdc9d5fc0b8c58e0e558506cffe7c08918e217d259a723637a julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet +aaf0b99a25f0fde7a7d054ad5a562af1f080a84ed8d750ce5ad9741c00bfcb9a julia/rewrite/arrow-rs/arrow-rs-list-rule3_v1.parquet +8332c983b137e04d84bec5a7d46f3f5b8270de65db2e3062ab87896b4c7ef744 julia/rewrite/arrow-rs/arrow-rs-list-rule3_v2.parquet +c32444882e1fa633aa3411449d417d904afb70337907c80503f998dfe0a943cd julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v1.parquet +30e3bea2ba646ab7bf0a45536904425e6f41478beddd60fbfa72eb1c6695ecac julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v2.parquet +19cc449cde41bac260b57b46b425a8225e456274915580eafb51182b86b8eded julia/rewrite/parquet-java/list_direct_map.v1.parquet +b4f7a5818830bbfa3a3a8fe935fcfa645889e9a2ac9f3615e6da8b98c36e4595 julia/rewrite/parquet-java/list_direct_map.v2.parquet +df6e405e214e651df86034c4b67b0720a3d71d8eccfcff1b75ce0c8397a4bd8a julia/rewrite/parquet-java/list_direct_map_utf8.v1.parquet +7bcbd22ad1305d5d2d67ed0143f8c150458951f4a1a08d9df446f23424768180 julia/rewrite/parquet-java/list_direct_map_utf8.v2.parquet +ac53fc3e511a056e0a466ce5e2b8be4c79d73e3871dd2f397155aae3e59f7961 julia/rewrite/parquet-java/list_rule1_primitive.v1.parquet +6e67250b399d2cc718478d01e9343866d188cd3663b7cfa4553e340266342670 julia/rewrite/parquet-java/list_rule1_primitive.v2.parquet +d1c1706a5bd289ae64ef34db86e3fd48fe56f1d67788736b050aa679ab3f4a67 julia/rewrite/parquet-java/list_rule2_struct.v1.parquet +67fc6cc8769e2baa9a86d5642794ca38b274eca4649fd67e50525cf51e4e6ecf julia/rewrite/parquet-java/list_rule2_struct.v2.parquet +847533a54c2687dd0eb849ac5e67e52dfd21e936eb9c65c5ff8bfa37e85ce87b julia/rewrite/parquet-java/list_rule3_nested.v1.parquet +6c5cff739e8349b593bd4f96fa8be3c4cf4a0fbcd8750776f42b08c100906323 julia/rewrite/parquet-java/list_rule3_nested.v2.parquet +c0cbc94cd2e842a3f997e163d61f002203522064a184707ea7bbaeef305e1a05 julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet +542231a9bc8d1cf49008eecc1c0fb7a4b94738dea837e7714b4e81b8f3ea24ac julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet +22e64d7cff4cb34d9bc980cb1380c2174ad6dda164ef26ce83985e2c2ab21e80 julia/rewrite/parquet-java/list_rule4_array.v1.parquet +d8ef8f67c22b8a33d92881055bce7592b312cb59f1fbc1f9af2d352eb0b4728c julia/rewrite/parquet-java/list_rule4_array.v2.parquet +b095cdd84f421ad1308f2aefd22e4d4e35fb1a3519d33c4eec74eee7a37ed5d8 julia/rewrite/parquet-java/list_rule4_tuple.v1.parquet +0c6f0006fbf1f8c0662bbabaa4c9c977f3c9ecf58fa7753d5c3a7777141d635e julia/rewrite/parquet-java/list_rule4_tuple.v2.parquet +831d4d653dfe80f07be8abb12cf996414e781df0df5a1e0870c487de0eda1507 julia/rewrite/parquet-java/list_rule5_optional_extended.v1.parquet +b9c556b6052ca645a65f952635fa3b0d0323848ddffeae4765e7da9f642a30c3 julia/rewrite/parquet-java/list_rule5_optional_extended.v2.parquet +f832162b8ffb40c4e2ab637367b67a606932b1665927f36ec6d33dfd4ea7e0fe julia/rewrite/parquet-java/list_rule5_optional_paired.v1.parquet +0802e8871fdc62b16c96ab1bfd6cc5ecb68c5186c8c556a893e7c81353a3e06e julia/rewrite/parquet-java/list_rule5_optional_paired.v2.parquet +32e8866097d622296b9130e9f53620241e848282372ca977238585d2f4addb3e julia/rewrite/parquet-java/list_rule5_required.v1.parquet +78ea66349d545dd653f18c59fc9c35630296ca0876bb46705e22c49688da0c79 julia/rewrite/parquet-java/list_rule5_required.v2.parquet +557584fabf79da29ac15d9eed78c875fa004557622e70db4532d8e92ce8854b5 julia/rewrite/parquet-java/map_arbitrary_names.v1.parquet +f4367ba8815f9b1eecd6b623ff769e6d90008e1597337dfc966dd1a45bced24b julia/rewrite/parquet-java/map_arbitrary_names.v2.parquet +2eed41707a21af87a89153ddce92d172ca438dd0ad3dd0f9e0ad23c8719b7f0f julia/rewrite/parquet-java/map_key_only.v1.parquet +243697559953c4c33790c2c6e03e24b57a8ed542fbc0b564bb885ad17721ac33 julia/rewrite/parquet-java/map_key_only.v2.parquet +96e8b27d829b8fabf7a381beefe6d14ece00b2652ae5bcaccb2238af87a4e872 julia/rewrite/parquet-java/map_standalone_mkv.v1.parquet +ea438ff2e20affac9ebd71d7ce1d1f4989fdde3e7d2d06477932a4fa3c5911ff julia/rewrite/parquet-java/map_standalone_mkv.v2.parquet +9de1ce6ad91d4ef7a29ffd4fc59e60b5ed6aa1fc7781ecd55c806002cdec93d3 julia/rewrite/parquet-java/map_standard.v1.parquet +e35ffcc7e60c2ab700724b57466e7daca3ca843e916fd2ba07ebabf6dd055e2e julia/rewrite/parquet-java/map_standard.v2.parquet +acf5536a16167a2a870dfd55ad2a5d208712069d8e0f034cf396fa6d6c193184 reference/external/arrow-rs/arrow-rs-duplicate-keys_v1.parquet +baaf9a09bc397db8c0b1d0453fa2d87e36f6a74bbc56778be39d27304007cbb3 reference/external/arrow-rs/arrow-rs-duplicate-keys_v2.parquet +40eb4521da12a9bd5db38cfe9e1f31dd91bd1095ec5b24b94f9a02318e40128f reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet +4fe0319ea46248612489d845eb6fdbfa127077f06a9e7a2efc14fa67def66ce1 reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet +916475c20dde6afe338d60e43d2e4a6293c2bc26efb6dd439d1d7d9299834bc6 reference/external/arrow-rs/arrow-rs-list-rule3_v1.parquet +ef72761c509f8bc302986d130f6d7d9382b975c8a3462a955c599bd5f427edac reference/external/arrow-rs/arrow-rs-list-rule3_v2.parquet +a33be66fe1a5e7189c457b49163e5a4b658228ffb5054a40cd22f20f731ba21a reference/external/arrow-rs/arrow-rs-optional-key-present_v1.parquet +f010b919bcc2d4e8c42f6d97df651266de06b6352f6b993756fd0a007e1c5028 reference/external/arrow-rs/arrow-rs-optional-key-present_v2.parquet +c84b4c582a00245acbc6330c80a3437f0007e1c2311adf31025b6952a95999e3 reference/external/parquet-java/list_direct_map.v1.parquet +c2d8ac178692160961f93b2a572a0bd4b19f71c63eddb56f7585c3e5a30c13eb reference/external/parquet-java/list_direct_map.v2.parquet +4f7faf3e2bef6c043d15e88feeb796728df51a9648abb81a301e0f6e6f97fd0b reference/external/parquet-java/list_direct_map_utf8.v1.parquet +edc66d338474404c9c7db8a4b93d5f1def987b4883f5aefe7d042c348aa282de reference/external/parquet-java/list_direct_map_utf8.v2.parquet +3466fa00347fae5832a8b751cd50d74e0af0df490d7262bdccf36f0c2f38c2db reference/external/parquet-java/list_rule1_primitive.v1.parquet +056989f7d112e735fc4f4e835278ba58610e522e1aa3a3e05afdaf540c0463cd reference/external/parquet-java/list_rule1_primitive.v2.parquet +b3c6ddd69b4b75bfe0ba1fcb21aca6db5d7b7c083fda0cd106b7229baa0a2f7b reference/external/parquet-java/list_rule2_struct.v1.parquet +2b059b3b5b375c736e7d0abb69a5f1e8dc48606008ffd9e3910f34d3543fc6a6 reference/external/parquet-java/list_rule2_struct.v2.parquet +c85821a7f6593ad3092f2efd3c8acc602f4969eb8dc36d49f1868b22f1d7d56a reference/external/parquet-java/list_rule3_nested.v1.parquet +5333c37c239ae57781c63eec972537eb428e275da8d3cca0634acb963989e73e reference/external/parquet-java/list_rule3_nested.v2.parquet +ab2c1a962ee992845540c6aa04056f15cdc4b40cf89627fb5160dd3fb07a5123 reference/external/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet +51ece6eccb1123e817c7e4a9f5f0ff235299501c7e7aa4e78f2878ecb4ae4e5f reference/external/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet +4b7a1f524f1f04a6fcb2f8ebabf1f82cf33d709cc154207e43e95ab61d09ed1a reference/external/parquet-java/list_rule4_array.v1.parquet +966d39ae1c7d1c011b3e2405ef0be9ff9d23372516abcc2060331b61c00bc642 reference/external/parquet-java/list_rule4_array.v2.parquet +2b7e4ae67a92182f44cff1c584096bc6b6d741106fe56eb20b0e2d3a41d0a170 reference/external/parquet-java/list_rule4_tuple.v1.parquet +9d51d23da84416b0b520e0fdb80796d80923b5c8cbdeeea454a8c422a897934a reference/external/parquet-java/list_rule4_tuple.v2.parquet +5281d05e2317fbc30e280c18b15078d0676b5ef22b1ee56d9788dc2e45d45af0 reference/external/parquet-java/list_rule5_optional_extended.v1.parquet +e2b4bca7e67bac4dafdca1a4aea01d3bdf764a6ece37419edd2c39699efec6c4 reference/external/parquet-java/list_rule5_optional_extended.v2.parquet +2873a69f47bd82a93625a78fda96a58a5783c5f473f0ace861216df876c9a725 reference/external/parquet-java/list_rule5_optional_paired.v1.parquet +a465d3f92575429b545d4edad4aece7dd30019860ee71af75f61d035c0b00f04 reference/external/parquet-java/list_rule5_optional_paired.v2.parquet +e16a41a44e974c44f8ca83021ecd55e016018bab7f86a930b2560af7d66de762 reference/external/parquet-java/list_rule5_required.v1.parquet +a730dbdfca94503ce0f1e6717f96c4636378b2b7f56d593bebae86fa86513127 reference/external/parquet-java/list_rule5_required.v2.parquet +0a6aadfd449407c0e6036d6b894e4f73c9d6a9b7a64b35eba16da2a685954ccb reference/external/parquet-java/map_arbitrary_names.v1.parquet +b69c7d382417e2be0834bbbc060a5a65548867c659aec51283b77650bccd8de4 reference/external/parquet-java/map_arbitrary_names.v2.parquet +e05e566da94fcdd96d5ca17c4f8f4403541a50c94002bef15639df889fb79102 reference/external/parquet-java/map_key_only.v1.parquet +3b6d03072c1046e000097bad8392319678256610a1ad56e7f6611c390cb0a5ef reference/external/parquet-java/map_key_only.v2.parquet +a99a78cc84bdc17a30d0ab73c28c5edf6a7640b85ceab9470fedaa7369a5f401 reference/external/parquet-java/map_standalone_mkv.v1.parquet +467cdc8abf6514d94aa8698d7b9bcad654684bd5d465cc0b973329a3c9db1f30 reference/external/parquet-java/map_standalone_mkv.v2.parquet +0acccffff97f7ff6867daaa35f57ad1a9609ed3407e81921a275f9c3f40d20d5 reference/external/parquet-java/map_standard.v1.parquet +136bf8f4949fa97f52b337b7a4c349d6fb2ca2f38e2df815adadea5df134c9fd reference/external/parquet-java/map_standard.v2.parquet +b804532d75ab1b2de0715321d3c9006b625e8e22b479ae423b9656ba7a0598bc reference/model/direct-list-of-map.v1.parquet +df227a3c97364237a4a769e3ddf851943bf879e84df4a7d347c694c7722fa83e reference/model/direct-list-of-map.v2.parquet +54a9e6c9bcb218b20cabfadbe8a5f54b5f247a363fd8f8a9d158f8abe8cbfc0f reference/model/list-rule-1.v1.parquet +bc5e6432b63dd5c3280fdaeed954005732d87164336b6a8e0da5ff78518ee354 reference/model/list-rule-1.v2.parquet +2992c963e21533f8a55ef2cc62e3187b40ca2ebf32321d677a394529238da861 reference/model/list-rule-2.v1.parquet +4a46cc1bee0ee24f405fa5585606f90e4fb5b021f1886c8268270ff9e080cd52 reference/model/list-rule-2.v2.parquet +d4b62a49c69f89938dae4d02c31e725f8362fe70a8cb3571a92423f8a210a121 reference/model/list-rule-3.v1.parquet +00f663c97d1f919ea16ee489ced5d53d23d8c0f8927351767ed43c4ecda3e15b reference/model/list-rule-3.v2.parquet +95924bc48dd8b746a1bc18d1819f4e4a164f37a342744fe57a3ce935c818e488 reference/model/list-rule-4-array.v1.parquet +e56cd1ec4bc6983c517c734c5e47486e4d491e010a8f2b631fe00d09f6df895d reference/model/list-rule-4-array.v2.parquet +1fb94f3f7cb694f148bbc0cd6de380f979e2479c9b31482250da8986de191507 reference/model/list-rule-4-tuple.v1.parquet +949a2102b8868e1feedf99657a8bcef7336b3025474ee67a464cdd0bb2a63a76 reference/model/list-rule-4-tuple.v2.parquet +590b16afb079231fda4193c6f5d921e74f1c9aee590de1422df04288aa111fca reference/model/list-rule-5-extended.v1.parquet +34a9335bea7be8eb96f562e53753cbc8e37a93e1611e14a5950fbb195e6ead29 reference/model/list-rule-5-extended.v2.parquet +b9feac1fa757c89acb9b5169c63eba043d41614922d5fa3c024b75510eafde88 reference/model/list-rule-5-paired.v1.parquet +60d8739a9e86d4d0f1d048018d0f8490d5427cc47e335f33ec014455ec625dc7 reference/model/list-rule-5-paired.v2.parquet +b355239079e012cd40660be9daa98ffcb53bd0eb34fb9752988aa65bd0031a26 reference/model/list-rule-5-required.v1.parquet +9c5173a38162e20ac8bd5bd6e416f02a5abe7839320f17f04b46d3b296169482 reference/model/list-rule-5-required.v2.parquet +8d4a2cd87b8c796252340035e38fd0031d954a97dc791d38b13ef2fe01e65aed reference/model/map-key-only.v1.parquet +299e6c8ea9ff0727100640869286a656566cedf6d99d899fe45773d58b87e871 reference/model/map-key-only.v2.parquet +9c7c47cebaae6a52117c6fe5b27a0be3184e99c9266e3d9dc5d65476def98c8b reference/model/map-optional-key.v1.parquet +77fab91a3365c936131bb9043524196f4ae7b838a90b473ce134c6673c3c6080 reference/model/map-optional-key.v2.parquet +813baf63ad2d7db577c7f03a28e45fc9c337f30745f0b6c68e9b166724430cd2 reference/model/map-standard.v1.parquet +951118322b172a4f705541e2c15db2cb0bcdf187994ca5f7b7426455d19af38e reference/model/map-standard.v2.parquet +9cc0752701d0c73bb0a665ee9ca6a2e684a142752167d1c40e18c97f7dac8c5a reference/model/schema-provenance.v1.parquet +3f446ba78cc7148159aaa8c64a2c1b352e398a4229264b2c5f61bfab232561e2 reference/model/schema-provenance.v2.parquet +4bfe8aa84c1b9e5a19301d0e0cbbf287f294826926cb21a1258f6debd5cb768f reference/property/generated-0002-v1.parquet +940ccc04ee257a01eb981819447e10ca3cfee65fdc3872c16e1d080ffb42f38a reference/property/generated-0004-v1.parquet +8576eb817601689e427a312c02a0e7855e7e40157bf56a1e70ab8db1ec1dffc8 reference/property/generated-0005-v2.parquet +486d23f725ad085dac9ffde5994834cce7bc7e4a1d14de18caa1889c5e163f28 reference/property/generated-0006-v1.parquet +ae446e076376986734b0a2a16bb263a340d92da7b823bcd2e8c8e3ab4b4b043e reference/property/generated-0007-v2.parquet +e7d42b3a568a8dd54dbe24a43bfdeaae84cca42338c849092f527a409ff294c9 reference/property/generated-0008-v1.parquet +1e93422dcb35e835859c17e427fd9c2eb08276f610f3c2ff70518e385f1050f7 reference/property/generated-0009-v2.parquet +5d307dc4b495ac7fdad9994cf96a829577263d49530a7627af792236403a0ca5 reference/property/generated-0010-v1.parquet +c5cb5721e9016388f52f4d153835cd3de477bccab63315e20e10381d6267095d reference/property/generated-0011-v2.parquet +e448fc79d6037dbdc73fdead587935915954d842fe6ff53abc41c7d335a15dea reference/property/generated-0014-v1.parquet +0845fd1a7e55072b52cfb58ce88e6fe88cb18546ce6dc80f4f49727564801e9e reference/property/generated-0015-v2.parquet +360680b09d595cdabcf0a8fd1a5e8d691edcb2d30e930829e402a4409ca9d84f reference/property/generated-0018-v1.parquet +5935851d2c41743fe70000fbf8e6232869aaac8953cbf952ed95bc7636f82513 reference/property/generated-0028-v1.parquet +205271023cc299692d03d0adf4e3fa77f1eccf5e34180103464361ca9bc015bd reference/property/generated-0034-v2.parquet +555285e98299279e909b1832cae273fb229ac10fe8dfade532a16240ddb7959a reference/property/generated-0038-v2.parquet +d06f6d79005c2d4697bf7cd4c72bb8d65e0d6e67674961985fb313919bea93f7 reference/property/generated-0048-v2.parquet +6cf9d1c250b1de9b8b728fddd9df949a667c98e48468211f063bfcb5eff6628f reference/property/generated-0058-v2.parquet +ad7085597ce597ae9dda7cc2d17ef826266ebe1e773aa2701057e441e8e69deb reference/property/generated-0062-v2.parquet +4e93e938a0d87c73c04887674fb77e38e3cc230bf43b42b2a2f73dbfbdbff095 reference/property/generated-0096-v2.parquet +9549669e5fa45926ccb32a251d151b043413f32d8c82a4c916da8b1f4fd91f02 reference/property/generated-0106-v2.parquet +ed92a45431f261d19369d0e802a8d377f5e0f30d8ade22f3e38d67ea8ffcec04 reference/property/generated-0107-v1.parquet +5b099ea68041ecc9326ca32bb384e6b185bd2c77f5dcdb33f5820fcaeec5d152 reference/property/generated-0117-v1.parquet +48e4673ffb51afd736278003f2da71b6b56d113a541c16b2684a36319fa784c9 reference/property/generated-0122-v2.parquet +bdf51ead923cbebf69df0bb1cdcc340a98b69e3fc7d83baf1b34a932cefa7a35 reference/property/generated-0123-v1.parquet +d7310128c56d92463cd54fe3906a9e1b02f0c4815e32e32390b827347fa1f69a reference/property/generated-0144-v2.parquet +58e975512b1d6266190f59f67c65b7bd0a39637ccecd4e5d01d152765ceefb52 reference/property/generated-0145-v1.parquet +3dcb776193896ccc5010d32b869b471470d5c0fae48c7e639647856d58f54774 reference/property/generated-0148-v2.parquet +00b731d38b4edf95178eec63f7b0abbf9b717417f7a7edd42f5778e7056adf09 reference/property/generated-0178-v1.parquet +565be1c5e017366f1504bf01007ac32418b20bf6113bd57b2da8d4c2c9ec1861 reference/property/generated-0191-v2.parquet +0fdef6d891b1731eaf217c63e336322f98aa29ced51a8b645335ace0bef2a5b4 reference/property/generated-0202-v2.parquet +c76283d134e73f6e7e74dcf868d79876973027280b66c9505a4b026f58c1f076 reference/property/generated-0249-v1.parquet +4e953527c320c16864803ca3d9b0063437d113c15be4bc5b458b2fdb8b49ba48 reference/property/generated-0251-v1.parquet diff --git a/test/conformance/n5/julia-fixtures/fixture-manifest.tsv b/test/conformance/n5/julia-fixtures/fixture-manifest.tsv new file mode 100644 index 0000000..fefe314 --- /dev/null +++ b/test/conformance/n5/julia-fixtures/fixture-manifest.tsv @@ -0,0 +1,257 @@ +kind case_id page_version codec reference target reference_sha256 target_sha256 +binding direct-list-of-map v1 uncompressed reference/model/direct-list-of-map.v1.parquet julia/model/direct-list-of-map.v1.parquet b804532d75ab1b2de0715321d3c9006b625e8e22b479ae423b9656ba7a0598bc 3cd92d1a3e45c93321933efe31b62d29fa7278c134497da08597baa765f129c4 +binding direct-list-of-map v2 uncompressed reference/model/direct-list-of-map.v2.parquet julia/model/direct-list-of-map.v2.parquet df227a3c97364237a4a769e3ddf851943bf879e84df4a7d347c694c7722fa83e 55eb313494bcc4b697ef0af13fc1a6522afbff67f8f42d4931aa86f09987fb08 +binding list-rule-1 v1 uncompressed reference/model/list-rule-1.v1.parquet julia/model/list-rule-1.v1.parquet 54a9e6c9bcb218b20cabfadbe8a5f54b5f247a363fd8f8a9d158f8abe8cbfc0f 1501150d96e2efa9114e74b1f0554ac038674f92999babffe25f0699e2720bc1 +binding list-rule-1 v2 uncompressed reference/model/list-rule-1.v2.parquet julia/model/list-rule-1.v2.parquet bc5e6432b63dd5c3280fdaeed954005732d87164336b6a8e0da5ff78518ee354 320a07c8ad7cb61837e40d1c2c8f79c1201cb826ddda5a1a01b8d020aa2c19e0 +binding list-rule-2 v1 uncompressed reference/model/list-rule-2.v1.parquet julia/model/list-rule-2.v1.parquet 2992c963e21533f8a55ef2cc62e3187b40ca2ebf32321d677a394529238da861 7f086c31f18f378e53a68e6baf9a4c987ff77c2ffcfc406d9e1e686d79ee168b +binding list-rule-2 v2 uncompressed reference/model/list-rule-2.v2.parquet julia/model/list-rule-2.v2.parquet 4a46cc1bee0ee24f405fa5585606f90e4fb5b021f1886c8268270ff9e080cd52 8de4b568bb8702f35ea07ef5bf17a0c3f0a61481d7bc8e26e8089bd30f9e8e5e +binding list-rule-3 v1 uncompressed reference/model/list-rule-3.v1.parquet julia/model/list-rule-3.v1.parquet d4b62a49c69f89938dae4d02c31e725f8362fe70a8cb3571a92423f8a210a121 b48b255f20ef9b42ee43d750e74c3c443029cc710fd3779702491519c2365789 +binding list-rule-3 v2 uncompressed reference/model/list-rule-3.v2.parquet julia/model/list-rule-3.v2.parquet 00f663c97d1f919ea16ee489ced5d53d23d8c0f8927351767ed43c4ecda3e15b 7976722c0954843510ff9b63e0cd49b5f2893a9e7589b776b4654143d46301b9 +binding list-rule-4-array v1 uncompressed reference/model/list-rule-4-array.v1.parquet julia/model/list-rule-4-array.v1.parquet 95924bc48dd8b746a1bc18d1819f4e4a164f37a342744fe57a3ce935c818e488 987ebdaa8f335dfde3e5d39f16e335642d801335a97cd77add9832cb0dd0230b +binding list-rule-4-array v2 uncompressed reference/model/list-rule-4-array.v2.parquet julia/model/list-rule-4-array.v2.parquet e56cd1ec4bc6983c517c734c5e47486e4d491e010a8f2b631fe00d09f6df895d 67864226a1f3cea4c9fec7ca55a6d77ed0a7ed159e35fc106756e3217a84ea01 +binding list-rule-4-tuple v1 uncompressed reference/model/list-rule-4-tuple.v1.parquet julia/model/list-rule-4-tuple.v1.parquet 1fb94f3f7cb694f148bbc0cd6de380f979e2479c9b31482250da8986de191507 0c996d1f5fefd0bac9a2288b536e97d98def99f24aa69122e4f7869ea0fd1510 +binding list-rule-4-tuple v2 uncompressed reference/model/list-rule-4-tuple.v2.parquet julia/model/list-rule-4-tuple.v2.parquet 949a2102b8868e1feedf99657a8bcef7336b3025474ee67a464cdd0bb2a63a76 2f6ac7955f51ac3cb23d88ffddb41d7dfca3a99fc91e42ccbb0c6ed531c28f0c +binding list-rule-5-extended v1 uncompressed reference/model/list-rule-5-extended.v1.parquet julia/model/list-rule-5-extended.v1.parquet 590b16afb079231fda4193c6f5d921e74f1c9aee590de1422df04288aa111fca 06b0b4672e66b30ccd8f2bc9d2d19d8aa61c897b7295ab4e6f2626c7c7fe9b88 +binding list-rule-5-extended v2 uncompressed reference/model/list-rule-5-extended.v2.parquet julia/model/list-rule-5-extended.v2.parquet 34a9335bea7be8eb96f562e53753cbc8e37a93e1611e14a5950fbb195e6ead29 da7da6bdc76cb788e7ab6bcf53d8c3fdb6e5b72ddf52d3b2db1b0579488a1e65 +binding list-rule-5-paired v1 uncompressed reference/model/list-rule-5-paired.v1.parquet julia/model/list-rule-5-paired.v1.parquet b9feac1fa757c89acb9b5169c63eba043d41614922d5fa3c024b75510eafde88 b3891b7e0f0f78e1259a10eee6c60e7b37339b9b1a40da91c58ec231ef578f1e +binding list-rule-5-paired v2 uncompressed reference/model/list-rule-5-paired.v2.parquet julia/model/list-rule-5-paired.v2.parquet 60d8739a9e86d4d0f1d048018d0f8490d5427cc47e335f33ec014455ec625dc7 3c771ccaab1e33f342ee99aa05b68c556722b1fbda86dea87f11de9d842feeeb +binding list-rule-5-required v1 uncompressed reference/model/list-rule-5-required.v1.parquet julia/model/list-rule-5-required.v1.parquet b355239079e012cd40660be9daa98ffcb53bd0eb34fb9752988aa65bd0031a26 2ef6f689de0f5590e6c2bdbc42ce49682aa69ffdbe62903adb4c0c83d1600641 +binding list-rule-5-required v2 uncompressed reference/model/list-rule-5-required.v2.parquet julia/model/list-rule-5-required.v2.parquet 9c5173a38162e20ac8bd5bd6e416f02a5abe7839320f17f04b46d3b296169482 099aef2c8d8de2d4abae1b225fd67e0d7f9005a50d262d6fb61a7f9253e0002d +binding map-key-only v1 uncompressed reference/model/map-key-only.v1.parquet julia/model/map-key-only.v1.parquet 8d4a2cd87b8c796252340035e38fd0031d954a97dc791d38b13ef2fe01e65aed beb097ba4292c9b54928442895edbd709e342146fff39e57564ba1f7292d8f23 +binding map-key-only v2 uncompressed reference/model/map-key-only.v2.parquet julia/model/map-key-only.v2.parquet 299e6c8ea9ff0727100640869286a656566cedf6d99d899fe45773d58b87e871 cd7eea07de636d91e381e263c230652e68c0ab06b726e00edea1f40491dc246c +binding map-optional-key v1 uncompressed reference/model/map-optional-key.v1.parquet julia/model/map-optional-key.v1.parquet 9c7c47cebaae6a52117c6fe5b27a0be3184e99c9266e3d9dc5d65476def98c8b a695dca0ca686f153b0fe3636f0baa1eff8a209df80f69a8d116a77d1c5e82cf +binding map-optional-key v2 uncompressed reference/model/map-optional-key.v2.parquet julia/model/map-optional-key.v2.parquet 77fab91a3365c936131bb9043524196f4ae7b838a90b473ce134c6673c3c6080 f866646481c7eccca7948d801c0e0fdc0f67447cce2604ee28c0a5e96f513501 +binding map-standard v1 uncompressed reference/model/map-standard.v1.parquet julia/model/map-standard.v1.parquet 813baf63ad2d7db577c7f03a28e45fc9c337f30745f0b6c68e9b166724430cd2 1780198fdd9764e85e17ec2a78a56cc27940657087a5f81cf7c15321d6ac74cc +binding map-standard v2 uncompressed reference/model/map-standard.v2.parquet julia/model/map-standard.v2.parquet 951118322b172a4f705541e2c15db2cb0bcdf187994ca5f7b7426455d19af38e 356c5e36c7e69b1074a48e2a920255accf2cf0f6d78a6bad46dc6b61cd630e07 +provenance schema-provenance v1 uncompressed reference/model/schema-provenance.v1.parquet julia/model/schema-provenance.v1.parquet 9cc0752701d0c73bb0a665ee9ca6a2e684a142752167d1c40e18c97f7dac8c5a 989a46daaeefedba745c885541d9e4d0dd703360f49b98736d6f27efaca45257 +provenance schema-provenance v2 uncompressed reference/model/schema-provenance.v2.parquet julia/model/schema-provenance.v2.parquet 3f446ba78cc7148159aaa8c64a2c1b352e398a4229264b2c5f61bfab232561e2 0b95c7dcc128d829c9b07c0a7db15adc8de720da8b40516a798aab46f10305b3 +property 2 v1 brotli reference/property/generated-0002-v1.parquet julia/property/generated-0002-v1-brotli.parquet 4bfe8aa84c1b9e5a19301d0e0cbbf287f294826926cb21a1258f6debd5cb768f 4c52263c0bc6f3eb7598aaf8f0c2038531bc7f847f0e8a45bad77cce68f4ed35 +property 2 v1 gzip reference/property/generated-0002-v1.parquet julia/property/generated-0002-v1-gzip.parquet 4bfe8aa84c1b9e5a19301d0e0cbbf287f294826926cb21a1258f6debd5cb768f bd3d065e0314c97806cc59c84ec4740a49c0a960fbc01d10b487f44be333e9bd +property 2 v1 lz4_raw reference/property/generated-0002-v1.parquet julia/property/generated-0002-v1-lz4_raw.parquet 4bfe8aa84c1b9e5a19301d0e0cbbf287f294826926cb21a1258f6debd5cb768f 6a8af5309e1185c00615773ec5c578dbc2407700ce52659a74affb41a2edde05 +property 2 v1 snappy reference/property/generated-0002-v1.parquet julia/property/generated-0002-v1-snappy.parquet 4bfe8aa84c1b9e5a19301d0e0cbbf287f294826926cb21a1258f6debd5cb768f e0b67b9145f0dd5d223a540e40237772891e40c692131a48df3f769a9fe42822 +property 2 v1 uncompressed reference/property/generated-0002-v1.parquet julia/property/generated-0002-v1-uncompressed.parquet 4bfe8aa84c1b9e5a19301d0e0cbbf287f294826926cb21a1258f6debd5cb768f d95a08cf23ec4ebe971b4874de5460a71b43baed2ddcae900ee13aefbd8df6d1 +property 2 v1 zstd reference/property/generated-0002-v1.parquet julia/property/generated-0002-v1-zstd.parquet 4bfe8aa84c1b9e5a19301d0e0cbbf287f294826926cb21a1258f6debd5cb768f dadebc22273aa2dc47b12fa4a4386563135d82b54cfaac7ef31af3653a7e7fd7 +property 4 v1 brotli reference/property/generated-0004-v1.parquet julia/property/generated-0004-v1-brotli.parquet 940ccc04ee257a01eb981819447e10ca3cfee65fdc3872c16e1d080ffb42f38a 470f2a6d2f5c5f6949def9205e7146ac8acd5249b343065447098250ecce605a +property 4 v1 gzip reference/property/generated-0004-v1.parquet julia/property/generated-0004-v1-gzip.parquet 940ccc04ee257a01eb981819447e10ca3cfee65fdc3872c16e1d080ffb42f38a ca2d2026b4947a2daa5f38b713b70824107e8d03b900899209ff00c3c0e87bce +property 4 v1 lz4_raw reference/property/generated-0004-v1.parquet julia/property/generated-0004-v1-lz4_raw.parquet 940ccc04ee257a01eb981819447e10ca3cfee65fdc3872c16e1d080ffb42f38a b6e96c0d2140294f40e44efbd7d93aecae1b3d56bba0c022f0b937902425d6ca +property 4 v1 snappy reference/property/generated-0004-v1.parquet julia/property/generated-0004-v1-snappy.parquet 940ccc04ee257a01eb981819447e10ca3cfee65fdc3872c16e1d080ffb42f38a 9604c353851f32cfa15e7c579b5e560ff0dbefa3958ee348f1e7431bbaaa310f +property 4 v1 uncompressed reference/property/generated-0004-v1.parquet julia/property/generated-0004-v1-uncompressed.parquet 940ccc04ee257a01eb981819447e10ca3cfee65fdc3872c16e1d080ffb42f38a 6b7aa5127af8b21e107e42cfdc6324152965ed4637ac939866216da981e695f6 +property 4 v1 zstd reference/property/generated-0004-v1.parquet julia/property/generated-0004-v1-zstd.parquet 940ccc04ee257a01eb981819447e10ca3cfee65fdc3872c16e1d080ffb42f38a fb80f46aa51f7b5b1ee8d2e935f4329bc130306067e0d1d572dc618b51eeb9dd +property 5 v2 brotli reference/property/generated-0005-v2.parquet julia/property/generated-0005-v2-brotli.parquet 8576eb817601689e427a312c02a0e7855e7e40157bf56a1e70ab8db1ec1dffc8 d580a983d9080ffe1a4f1c639c07f95b72f4a78a7fd8943b31388f3db790d752 +property 5 v2 gzip reference/property/generated-0005-v2.parquet julia/property/generated-0005-v2-gzip.parquet 8576eb817601689e427a312c02a0e7855e7e40157bf56a1e70ab8db1ec1dffc8 6cd0c6381540b6f81ac6fd041505c712f003fd476b664ba5ec61f4db35d77957 +property 5 v2 lz4_raw reference/property/generated-0005-v2.parquet julia/property/generated-0005-v2-lz4_raw.parquet 8576eb817601689e427a312c02a0e7855e7e40157bf56a1e70ab8db1ec1dffc8 b3271eb65b2c955448628399e55480994aa838f0ff011d7e09498c4c10cf4835 +property 5 v2 snappy reference/property/generated-0005-v2.parquet julia/property/generated-0005-v2-snappy.parquet 8576eb817601689e427a312c02a0e7855e7e40157bf56a1e70ab8db1ec1dffc8 0ac610ab2f5dcfefc7290f4e7f045cfcf398e9e7de80554e5c075a45a9f148d7 +property 5 v2 uncompressed reference/property/generated-0005-v2.parquet julia/property/generated-0005-v2-uncompressed.parquet 8576eb817601689e427a312c02a0e7855e7e40157bf56a1e70ab8db1ec1dffc8 90224b72fc575a9fe620b93b11760e187ff7cf117a5641a08fdfb111444a34b7 +property 5 v2 zstd reference/property/generated-0005-v2.parquet julia/property/generated-0005-v2-zstd.parquet 8576eb817601689e427a312c02a0e7855e7e40157bf56a1e70ab8db1ec1dffc8 e1b01ac66e24fb286038cb74449d57717443482cffd5add212f7209187b61190 +property 6 v1 brotli reference/property/generated-0006-v1.parquet julia/property/generated-0006-v1-brotli.parquet 486d23f725ad085dac9ffde5994834cce7bc7e4a1d14de18caa1889c5e163f28 3229de621e58063dc8e54e7917bccdbdb2f5f7862f67389661d54adebf1a1f4f +property 6 v1 gzip reference/property/generated-0006-v1.parquet julia/property/generated-0006-v1-gzip.parquet 486d23f725ad085dac9ffde5994834cce7bc7e4a1d14de18caa1889c5e163f28 5e2a7e6c915bbbdbbfaaab840898de90c9677c4fdd25cc9b08aaf1397ef50876 +property 6 v1 lz4_raw reference/property/generated-0006-v1.parquet julia/property/generated-0006-v1-lz4_raw.parquet 486d23f725ad085dac9ffde5994834cce7bc7e4a1d14de18caa1889c5e163f28 1d2a6e0763caa72fde602cf3a5158fb9fe1b77a487723780135b26b5858181be +property 6 v1 snappy reference/property/generated-0006-v1.parquet julia/property/generated-0006-v1-snappy.parquet 486d23f725ad085dac9ffde5994834cce7bc7e4a1d14de18caa1889c5e163f28 0cdd2e50493ff1786fd07e69d6e039e3acf22cac39413b40f535d6fad0df681d +property 6 v1 uncompressed reference/property/generated-0006-v1.parquet julia/property/generated-0006-v1-uncompressed.parquet 486d23f725ad085dac9ffde5994834cce7bc7e4a1d14de18caa1889c5e163f28 f0a8b7e3ef263d562ebb4ebdb778628654e7632dc3a6ba00fdb2dff4e93a3d4d +property 6 v1 zstd reference/property/generated-0006-v1.parquet julia/property/generated-0006-v1-zstd.parquet 486d23f725ad085dac9ffde5994834cce7bc7e4a1d14de18caa1889c5e163f28 af2509a05c749f4c3b46e3815c51f6196fb45baea4cb232866af38fa4fcd623a +property 7 v2 brotli reference/property/generated-0007-v2.parquet julia/property/generated-0007-v2-brotli.parquet ae446e076376986734b0a2a16bb263a340d92da7b823bcd2e8c8e3ab4b4b043e ecd60078edb9b7d8c742d95eb584387ead98d348905588d77049ee2ddb3c63e4 +property 7 v2 gzip reference/property/generated-0007-v2.parquet julia/property/generated-0007-v2-gzip.parquet ae446e076376986734b0a2a16bb263a340d92da7b823bcd2e8c8e3ab4b4b043e 9941876a4d0ed9ad6325662d5065a91b4fed9f75f482ab5a0f10d4243401c515 +property 7 v2 lz4_raw reference/property/generated-0007-v2.parquet julia/property/generated-0007-v2-lz4_raw.parquet ae446e076376986734b0a2a16bb263a340d92da7b823bcd2e8c8e3ab4b4b043e 8db9aa30605d421f480e5b6e18cfab03054385fd59f013c10fb39e0716a76384 +property 7 v2 snappy reference/property/generated-0007-v2.parquet julia/property/generated-0007-v2-snappy.parquet ae446e076376986734b0a2a16bb263a340d92da7b823bcd2e8c8e3ab4b4b043e ce35740658a531d6fa65cf36286036e75454d0f3ab7c0fd48db3da2668b2e4fd +property 7 v2 uncompressed reference/property/generated-0007-v2.parquet julia/property/generated-0007-v2-uncompressed.parquet ae446e076376986734b0a2a16bb263a340d92da7b823bcd2e8c8e3ab4b4b043e 73e0b2f3aeca0fd57416ee6e1f1242baa3752ae267c79089e3800dd61044e706 +property 7 v2 zstd reference/property/generated-0007-v2.parquet julia/property/generated-0007-v2-zstd.parquet ae446e076376986734b0a2a16bb263a340d92da7b823bcd2e8c8e3ab4b4b043e aa271051d84a3fefd09655260c488d41858c908d2112de6e54f93ac32eda2036 +property 8 v1 brotli reference/property/generated-0008-v1.parquet julia/property/generated-0008-v1-brotli.parquet e7d42b3a568a8dd54dbe24a43bfdeaae84cca42338c849092f527a409ff294c9 6ea0ad9396016feed334f6faac37925c2eb63872bbb3a56c45bd6464633b3cc4 +property 8 v1 gzip reference/property/generated-0008-v1.parquet julia/property/generated-0008-v1-gzip.parquet e7d42b3a568a8dd54dbe24a43bfdeaae84cca42338c849092f527a409ff294c9 e38c60bd13289a9480aaec1edf879c0d1925c09ae3e1bfec37cae1faf35f85f9 +property 8 v1 lz4_raw reference/property/generated-0008-v1.parquet julia/property/generated-0008-v1-lz4_raw.parquet e7d42b3a568a8dd54dbe24a43bfdeaae84cca42338c849092f527a409ff294c9 969ccaa5870c17908abd06181f71e7786a105b02663f1d57eea8e1fe5c99e09f +property 8 v1 snappy reference/property/generated-0008-v1.parquet julia/property/generated-0008-v1-snappy.parquet e7d42b3a568a8dd54dbe24a43bfdeaae84cca42338c849092f527a409ff294c9 2d401978d38b2c0dd63b1ad1b9872854bac561414981cf0dc30c7cf3ff7991ee +property 8 v1 uncompressed reference/property/generated-0008-v1.parquet julia/property/generated-0008-v1-uncompressed.parquet e7d42b3a568a8dd54dbe24a43bfdeaae84cca42338c849092f527a409ff294c9 5174cba3dd2096381491fd47bf6f9a497f2640a05f803a95eb1f0bd29c48598d +property 8 v1 zstd reference/property/generated-0008-v1.parquet julia/property/generated-0008-v1-zstd.parquet e7d42b3a568a8dd54dbe24a43bfdeaae84cca42338c849092f527a409ff294c9 c0203aeef6ae8d7e6e88f9319e552bcbec13f79809cf79fb658171eff3a3812e +property 9 v2 brotli reference/property/generated-0009-v2.parquet julia/property/generated-0009-v2-brotli.parquet 1e93422dcb35e835859c17e427fd9c2eb08276f610f3c2ff70518e385f1050f7 62f0915ca7411dc35e906d0e41fab32e09ba70eadf7bdb122760609a4c533b26 +property 9 v2 gzip reference/property/generated-0009-v2.parquet julia/property/generated-0009-v2-gzip.parquet 1e93422dcb35e835859c17e427fd9c2eb08276f610f3c2ff70518e385f1050f7 b483b5cedfaf16edf67201a9d7cf13ff20b87fdb119f72b33ab4cda7ee0d3c3f +property 9 v2 lz4_raw reference/property/generated-0009-v2.parquet julia/property/generated-0009-v2-lz4_raw.parquet 1e93422dcb35e835859c17e427fd9c2eb08276f610f3c2ff70518e385f1050f7 df387db6ef217af1b9025b21262965a608d16a5fa02296f52630f52ad06b4654 +property 9 v2 snappy reference/property/generated-0009-v2.parquet julia/property/generated-0009-v2-snappy.parquet 1e93422dcb35e835859c17e427fd9c2eb08276f610f3c2ff70518e385f1050f7 906d43803d0c4939cc5e3477cc997fadf8b25f1a9c061bda7820ae30ef88f9d4 +property 9 v2 uncompressed reference/property/generated-0009-v2.parquet julia/property/generated-0009-v2-uncompressed.parquet 1e93422dcb35e835859c17e427fd9c2eb08276f610f3c2ff70518e385f1050f7 e26b53294981c361d1cd202c287a7dab38f3134e93a72dd64c0bba785609353d +property 9 v2 zstd reference/property/generated-0009-v2.parquet julia/property/generated-0009-v2-zstd.parquet 1e93422dcb35e835859c17e427fd9c2eb08276f610f3c2ff70518e385f1050f7 08aba081ff45baa57fb4b3a5f0c3309d8902e8f268acf70737587941c0fce73f +property 10 v1 brotli reference/property/generated-0010-v1.parquet julia/property/generated-0010-v1-brotli.parquet 5d307dc4b495ac7fdad9994cf96a829577263d49530a7627af792236403a0ca5 86177f80c869a423c3463e4f4d79fd7ef1c4c20afa11ba4fa58685a938209395 +property 10 v1 gzip reference/property/generated-0010-v1.parquet julia/property/generated-0010-v1-gzip.parquet 5d307dc4b495ac7fdad9994cf96a829577263d49530a7627af792236403a0ca5 c2e77b5d267bf2d15dbf936577706c4a66d6852fb9b147105cee95cfbb430492 +property 10 v1 lz4_raw reference/property/generated-0010-v1.parquet julia/property/generated-0010-v1-lz4_raw.parquet 5d307dc4b495ac7fdad9994cf96a829577263d49530a7627af792236403a0ca5 54f15384ca731da3f6c143d7bf9fabbc6b08f2d9a61671391caf7479f9a4c37b +property 10 v1 snappy reference/property/generated-0010-v1.parquet julia/property/generated-0010-v1-snappy.parquet 5d307dc4b495ac7fdad9994cf96a829577263d49530a7627af792236403a0ca5 3926fb9bb0c833f3a1ea3240c8016a96b82e4e423701df30a77d5de70bca65df +property 10 v1 uncompressed reference/property/generated-0010-v1.parquet julia/property/generated-0010-v1-uncompressed.parquet 5d307dc4b495ac7fdad9994cf96a829577263d49530a7627af792236403a0ca5 fd0206f9f2599ddf38a8fb91ed518996a56155f172823b385ed2b66a3c191be4 +property 10 v1 zstd reference/property/generated-0010-v1.parquet julia/property/generated-0010-v1-zstd.parquet 5d307dc4b495ac7fdad9994cf96a829577263d49530a7627af792236403a0ca5 6889010559e0b875e7d4208de19afaeb60a9adad781c134f5ef716c668138fb0 +property 11 v2 brotli reference/property/generated-0011-v2.parquet julia/property/generated-0011-v2-brotli.parquet c5cb5721e9016388f52f4d153835cd3de477bccab63315e20e10381d6267095d 329e5c69bf4cf4e59ca2d87fb93d1068afbe14a55db743c7fd3ea9ee6d2b1dc8 +property 11 v2 gzip reference/property/generated-0011-v2.parquet julia/property/generated-0011-v2-gzip.parquet c5cb5721e9016388f52f4d153835cd3de477bccab63315e20e10381d6267095d ba241f6ec00c55a98864886bc32c5494931fb94b999f31ef7051d2d63a96d1b3 +property 11 v2 lz4_raw reference/property/generated-0011-v2.parquet julia/property/generated-0011-v2-lz4_raw.parquet c5cb5721e9016388f52f4d153835cd3de477bccab63315e20e10381d6267095d e9f3ea6f819b22b37661a398fde0f16610750ae8d38e187dfbc134f1cddbd158 +property 11 v2 snappy reference/property/generated-0011-v2.parquet julia/property/generated-0011-v2-snappy.parquet c5cb5721e9016388f52f4d153835cd3de477bccab63315e20e10381d6267095d 090ea05ae46f5dd45b3d858e593af5a5c73785319378f1b207aa6d2680ada1db +property 11 v2 uncompressed reference/property/generated-0011-v2.parquet julia/property/generated-0011-v2-uncompressed.parquet c5cb5721e9016388f52f4d153835cd3de477bccab63315e20e10381d6267095d ff6992972160170aff728ff3f7c53473c4190e64bc53e315128194799969fca8 +property 11 v2 zstd reference/property/generated-0011-v2.parquet julia/property/generated-0011-v2-zstd.parquet c5cb5721e9016388f52f4d153835cd3de477bccab63315e20e10381d6267095d 0196152849dd81597908abec7588bceea7b6f3d4e3eb6e18e427fbec846aa15a +property 14 v1 brotli reference/property/generated-0014-v1.parquet julia/property/generated-0014-v1-brotli.parquet e448fc79d6037dbdc73fdead587935915954d842fe6ff53abc41c7d335a15dea 1bd5c130fe288087daf514e8acfff9b24e4dc0d8a41af903b97dc7e9551cb2fd +property 14 v1 gzip reference/property/generated-0014-v1.parquet julia/property/generated-0014-v1-gzip.parquet e448fc79d6037dbdc73fdead587935915954d842fe6ff53abc41c7d335a15dea d46975dbaa3bd7f5c9335e2a7afecc1c51018afcf49973d5fa7436630ae0ee73 +property 14 v1 lz4_raw reference/property/generated-0014-v1.parquet julia/property/generated-0014-v1-lz4_raw.parquet e448fc79d6037dbdc73fdead587935915954d842fe6ff53abc41c7d335a15dea 55cf0a1b8c936ae0b5e0409b8cb6e3501404a5821dcf473906787d27836ca669 +property 14 v1 snappy reference/property/generated-0014-v1.parquet julia/property/generated-0014-v1-snappy.parquet e448fc79d6037dbdc73fdead587935915954d842fe6ff53abc41c7d335a15dea 7b0a73d0df98bb54e2071b920eebd00053871c2f273c98d4970114de0c53984e +property 14 v1 uncompressed reference/property/generated-0014-v1.parquet julia/property/generated-0014-v1-uncompressed.parquet e448fc79d6037dbdc73fdead587935915954d842fe6ff53abc41c7d335a15dea 01bba38899a61aca7329887710a9d073972fe8f2de01ae522b33d7932bb79f63 +property 14 v1 zstd reference/property/generated-0014-v1.parquet julia/property/generated-0014-v1-zstd.parquet e448fc79d6037dbdc73fdead587935915954d842fe6ff53abc41c7d335a15dea a522c5284b3a6593a47f30b9cfc1402c66f5b661164b6dbe81c277ee0a86cc7a +property 15 v2 brotli reference/property/generated-0015-v2.parquet julia/property/generated-0015-v2-brotli.parquet 0845fd1a7e55072b52cfb58ce88e6fe88cb18546ce6dc80f4f49727564801e9e cff5c741b35a9cdfda79e4fb53df6ccad367d30ac4a59d87714c7aabf12d756f +property 15 v2 gzip reference/property/generated-0015-v2.parquet julia/property/generated-0015-v2-gzip.parquet 0845fd1a7e55072b52cfb58ce88e6fe88cb18546ce6dc80f4f49727564801e9e 94b22cb22ac553d25006924d51ec98a403884eed256a0d45435334f664781f7d +property 15 v2 lz4_raw reference/property/generated-0015-v2.parquet julia/property/generated-0015-v2-lz4_raw.parquet 0845fd1a7e55072b52cfb58ce88e6fe88cb18546ce6dc80f4f49727564801e9e 028acbdbc55150c27b2a90db12b38cd0054c39c50c97d086bea222956b0f0169 +property 15 v2 snappy reference/property/generated-0015-v2.parquet julia/property/generated-0015-v2-snappy.parquet 0845fd1a7e55072b52cfb58ce88e6fe88cb18546ce6dc80f4f49727564801e9e 66d06e79d863df5c4137a976470ea8c98df428637f90a5796a90147712b12f66 +property 15 v2 uncompressed reference/property/generated-0015-v2.parquet julia/property/generated-0015-v2-uncompressed.parquet 0845fd1a7e55072b52cfb58ce88e6fe88cb18546ce6dc80f4f49727564801e9e c83e5973b6e3c1893b27731b96605ee2a947f4c4790a78be5b5dabb58230e3a5 +property 15 v2 zstd reference/property/generated-0015-v2.parquet julia/property/generated-0015-v2-zstd.parquet 0845fd1a7e55072b52cfb58ce88e6fe88cb18546ce6dc80f4f49727564801e9e e9afe072d5495ec59ecb64804bab60a03e4f28bd68c2d6813b8800c952e7ba6a +property 18 v1 brotli reference/property/generated-0018-v1.parquet julia/property/generated-0018-v1-brotli.parquet 360680b09d595cdabcf0a8fd1a5e8d691edcb2d30e930829e402a4409ca9d84f 4a1049e083efd4db62d1950ae55b60a02b1f11dd48926ae3ec6f9f5f7247b3a5 +property 18 v1 gzip reference/property/generated-0018-v1.parquet julia/property/generated-0018-v1-gzip.parquet 360680b09d595cdabcf0a8fd1a5e8d691edcb2d30e930829e402a4409ca9d84f ec54eeb5bd624d40de14a0648696b1e142fcb56cba73f9c0a27815cde72dd62d +property 18 v1 lz4_raw reference/property/generated-0018-v1.parquet julia/property/generated-0018-v1-lz4_raw.parquet 360680b09d595cdabcf0a8fd1a5e8d691edcb2d30e930829e402a4409ca9d84f e652771c3b56087db00ecb99a875d4b0d967be6f5b25c9d5634db3cec48eba9b +property 18 v1 snappy reference/property/generated-0018-v1.parquet julia/property/generated-0018-v1-snappy.parquet 360680b09d595cdabcf0a8fd1a5e8d691edcb2d30e930829e402a4409ca9d84f 5c2d52f8fb338c2d252a1156caeea089c8b3ae8a2e7aad410cb3224a8e2c31d3 +property 18 v1 uncompressed reference/property/generated-0018-v1.parquet julia/property/generated-0018-v1-uncompressed.parquet 360680b09d595cdabcf0a8fd1a5e8d691edcb2d30e930829e402a4409ca9d84f ac5e8e0371fd2f499f09df1aadea4648eab0f5aa3594729baceccb2360c404cd +property 18 v1 zstd reference/property/generated-0018-v1.parquet julia/property/generated-0018-v1-zstd.parquet 360680b09d595cdabcf0a8fd1a5e8d691edcb2d30e930829e402a4409ca9d84f 85381544858a2293267cf513d35934d16c5500c4b9c3ff29076097f410e970d7 +property 28 v1 brotli reference/property/generated-0028-v1.parquet julia/property/generated-0028-v1-brotli.parquet 5935851d2c41743fe70000fbf8e6232869aaac8953cbf952ed95bc7636f82513 129a0c7e6562926f4ab35506e26772f4a85bc6e1abcbbc418a52b9f759011007 +property 28 v1 gzip reference/property/generated-0028-v1.parquet julia/property/generated-0028-v1-gzip.parquet 5935851d2c41743fe70000fbf8e6232869aaac8953cbf952ed95bc7636f82513 76630e94386c56c1fe3cb3838bcefbb8f07a2721851cb370e45f493342adae76 +property 28 v1 lz4_raw reference/property/generated-0028-v1.parquet julia/property/generated-0028-v1-lz4_raw.parquet 5935851d2c41743fe70000fbf8e6232869aaac8953cbf952ed95bc7636f82513 7076ff328050bbd7233ad65dc96ed7180e5a9b1caee2197f63e278d73b555149 +property 28 v1 snappy reference/property/generated-0028-v1.parquet julia/property/generated-0028-v1-snappy.parquet 5935851d2c41743fe70000fbf8e6232869aaac8953cbf952ed95bc7636f82513 d66e711a68fc4bbe8e393dcb42ea2740fa01c3ac69df62fb02281c43a3b7c8f6 +property 28 v1 uncompressed reference/property/generated-0028-v1.parquet julia/property/generated-0028-v1-uncompressed.parquet 5935851d2c41743fe70000fbf8e6232869aaac8953cbf952ed95bc7636f82513 0f79f24eb9ac8c4a3972a18746c620ea574da27b48f661ce1396a8869afeade6 +property 28 v1 zstd reference/property/generated-0028-v1.parquet julia/property/generated-0028-v1-zstd.parquet 5935851d2c41743fe70000fbf8e6232869aaac8953cbf952ed95bc7636f82513 619c6b8655ec07a70ed2187c43e99465394222a7a8acf03467215e4a55c82c56 +property 34 v2 brotli reference/property/generated-0034-v2.parquet julia/property/generated-0034-v2-brotli.parquet 205271023cc299692d03d0adf4e3fa77f1eccf5e34180103464361ca9bc015bd 6edda09d48570e719691edf0227bcf05a5fda8da6a7eddbcc90ca0e5303713a9 +property 34 v2 gzip reference/property/generated-0034-v2.parquet julia/property/generated-0034-v2-gzip.parquet 205271023cc299692d03d0adf4e3fa77f1eccf5e34180103464361ca9bc015bd 62353c41290f30a739ebb19e3a62020990c537b1beb43bde1e627bbb60e84845 +property 34 v2 lz4_raw reference/property/generated-0034-v2.parquet julia/property/generated-0034-v2-lz4_raw.parquet 205271023cc299692d03d0adf4e3fa77f1eccf5e34180103464361ca9bc015bd 53257f2e29a58c1f4ac948597f8adeca32de74457b141a69220903494bbdb3ea +property 34 v2 snappy reference/property/generated-0034-v2.parquet julia/property/generated-0034-v2-snappy.parquet 205271023cc299692d03d0adf4e3fa77f1eccf5e34180103464361ca9bc015bd e22ebbae91d2795a7ddc66eb430155df7c6ce24dfd530f4960ce93e2d6137aca +property 34 v2 uncompressed reference/property/generated-0034-v2.parquet julia/property/generated-0034-v2-uncompressed.parquet 205271023cc299692d03d0adf4e3fa77f1eccf5e34180103464361ca9bc015bd 0da351162998efffaef53c035af079af72b952acea593b8f3554702b16a2e9e8 +property 34 v2 zstd reference/property/generated-0034-v2.parquet julia/property/generated-0034-v2-zstd.parquet 205271023cc299692d03d0adf4e3fa77f1eccf5e34180103464361ca9bc015bd 380cb85adb83b197eeb91c03a16c43c065e43ec04da394bb6822733735fba20e +property 38 v2 brotli reference/property/generated-0038-v2.parquet julia/property/generated-0038-v2-brotli.parquet 555285e98299279e909b1832cae273fb229ac10fe8dfade532a16240ddb7959a 5ad7fbf6fdb9235427ee68d580b8e324d7827a70f058d291309962cb8785c0c0 +property 38 v2 gzip reference/property/generated-0038-v2.parquet julia/property/generated-0038-v2-gzip.parquet 555285e98299279e909b1832cae273fb229ac10fe8dfade532a16240ddb7959a 027f0cc87a4b5de6ecd96ecf8231d55f3b208b5bbf0468a53680ae540b0c7644 +property 38 v2 lz4_raw reference/property/generated-0038-v2.parquet julia/property/generated-0038-v2-lz4_raw.parquet 555285e98299279e909b1832cae273fb229ac10fe8dfade532a16240ddb7959a c9f2d02e46cb65c9a1fd891a041f792e16139d6e8d69d3262ba8bffc932b7b03 +property 38 v2 snappy reference/property/generated-0038-v2.parquet julia/property/generated-0038-v2-snappy.parquet 555285e98299279e909b1832cae273fb229ac10fe8dfade532a16240ddb7959a e88612da352e14bf9068e0d3df599699ef431053685ccc249649d5c64ca96c08 +property 38 v2 uncompressed reference/property/generated-0038-v2.parquet julia/property/generated-0038-v2-uncompressed.parquet 555285e98299279e909b1832cae273fb229ac10fe8dfade532a16240ddb7959a 4e6abb4e0d6ceb60815129f1544bc0209a9a0f3d35bd99de647f73a299d108a9 +property 38 v2 zstd reference/property/generated-0038-v2.parquet julia/property/generated-0038-v2-zstd.parquet 555285e98299279e909b1832cae273fb229ac10fe8dfade532a16240ddb7959a bfb7d9529dfdcf3487fc460f97b6386920c728ecf30f41ff771cff5b0fe595bd +property 48 v2 brotli reference/property/generated-0048-v2.parquet julia/property/generated-0048-v2-brotli.parquet d06f6d79005c2d4697bf7cd4c72bb8d65e0d6e67674961985fb313919bea93f7 d4f57e1b5fc618c74813e6a90a21a21a8be246c1625364502825a90f58654d2c +property 48 v2 gzip reference/property/generated-0048-v2.parquet julia/property/generated-0048-v2-gzip.parquet d06f6d79005c2d4697bf7cd4c72bb8d65e0d6e67674961985fb313919bea93f7 95ac41365a6f5f49278443f03bb80b82a07431f426c1277f7d1b7cd20734430b +property 48 v2 lz4_raw reference/property/generated-0048-v2.parquet julia/property/generated-0048-v2-lz4_raw.parquet d06f6d79005c2d4697bf7cd4c72bb8d65e0d6e67674961985fb313919bea93f7 2ae57bf5188303172e1cc7e0155f2aa65aeed4c300a46bf0f6b7a85296922f1c +property 48 v2 snappy reference/property/generated-0048-v2.parquet julia/property/generated-0048-v2-snappy.parquet d06f6d79005c2d4697bf7cd4c72bb8d65e0d6e67674961985fb313919bea93f7 c2e56e1a6b10e43ff4f14c8f4a5fd3982a16c7a7b88652ab27ff3332f030e7a5 +property 48 v2 uncompressed reference/property/generated-0048-v2.parquet julia/property/generated-0048-v2-uncompressed.parquet d06f6d79005c2d4697bf7cd4c72bb8d65e0d6e67674961985fb313919bea93f7 9461f475bd285c22adc2c1a74c8d905b2022fd08a381d83623921e73091135e4 +property 48 v2 zstd reference/property/generated-0048-v2.parquet julia/property/generated-0048-v2-zstd.parquet d06f6d79005c2d4697bf7cd4c72bb8d65e0d6e67674961985fb313919bea93f7 c9b5bee8cfc84d0cc8d7ca9b2bdbbfc52b4628ebbd1f4ad7b88c76639d405c40 +property 58 v2 brotli reference/property/generated-0058-v2.parquet julia/property/generated-0058-v2-brotli.parquet 6cf9d1c250b1de9b8b728fddd9df949a667c98e48468211f063bfcb5eff6628f de4a8f6f02252f661c64b0c71d8e74987a6af57872cadf68d807841b47f27590 +property 58 v2 gzip reference/property/generated-0058-v2.parquet julia/property/generated-0058-v2-gzip.parquet 6cf9d1c250b1de9b8b728fddd9df949a667c98e48468211f063bfcb5eff6628f e1f65729ca9da5813586ad862f43e96ab4fcddecf65b9a16858961038b7c809c +property 58 v2 lz4_raw reference/property/generated-0058-v2.parquet julia/property/generated-0058-v2-lz4_raw.parquet 6cf9d1c250b1de9b8b728fddd9df949a667c98e48468211f063bfcb5eff6628f 41be1bdad62789707cc1a135a1893163e3f3bbbfcdb80f7b6c5e9487f1f53a65 +property 58 v2 snappy reference/property/generated-0058-v2.parquet julia/property/generated-0058-v2-snappy.parquet 6cf9d1c250b1de9b8b728fddd9df949a667c98e48468211f063bfcb5eff6628f b9ad0e2bf83e7ff1badf9b7458ebb001a016f0b6bfc3a29a3f3161d91299ede7 +property 58 v2 uncompressed reference/property/generated-0058-v2.parquet julia/property/generated-0058-v2-uncompressed.parquet 6cf9d1c250b1de9b8b728fddd9df949a667c98e48468211f063bfcb5eff6628f d43a2eb1090974bc470e7eea2a76841c821306ec995706b384725f51e9de5ea6 +property 58 v2 zstd reference/property/generated-0058-v2.parquet julia/property/generated-0058-v2-zstd.parquet 6cf9d1c250b1de9b8b728fddd9df949a667c98e48468211f063bfcb5eff6628f f2631a99b551973aa88dfaff2f2b54f18e7e43092b57865fb3e0ff656511e015 +property 62 v2 brotli reference/property/generated-0062-v2.parquet julia/property/generated-0062-v2-brotli.parquet ad7085597ce597ae9dda7cc2d17ef826266ebe1e773aa2701057e441e8e69deb 842b3b2d51d4d88f7231492971fe4f427010f3fb569970ee726021800e4aa5ee +property 62 v2 gzip reference/property/generated-0062-v2.parquet julia/property/generated-0062-v2-gzip.parquet ad7085597ce597ae9dda7cc2d17ef826266ebe1e773aa2701057e441e8e69deb 375161c057355f0237583dd1d60bca7d26da1b95e63a9aa6dd999085f4ff8323 +property 62 v2 lz4_raw reference/property/generated-0062-v2.parquet julia/property/generated-0062-v2-lz4_raw.parquet ad7085597ce597ae9dda7cc2d17ef826266ebe1e773aa2701057e441e8e69deb 035d23477076d704a8cfe3dc93ea8b2e89e36657bdb167dd39085578fb1c48d7 +property 62 v2 snappy reference/property/generated-0062-v2.parquet julia/property/generated-0062-v2-snappy.parquet ad7085597ce597ae9dda7cc2d17ef826266ebe1e773aa2701057e441e8e69deb 4652e73712b2907568922910ea4557886c250f3061a441da6e3207944a253e26 +property 62 v2 uncompressed reference/property/generated-0062-v2.parquet julia/property/generated-0062-v2-uncompressed.parquet ad7085597ce597ae9dda7cc2d17ef826266ebe1e773aa2701057e441e8e69deb a6a1d3edc66ea53e88e67aac480da9f6e7417bea1ca423f365706fd537320c84 +property 62 v2 zstd reference/property/generated-0062-v2.parquet julia/property/generated-0062-v2-zstd.parquet ad7085597ce597ae9dda7cc2d17ef826266ebe1e773aa2701057e441e8e69deb b72fb3e6d50dea4a9fa3ac38b4ece6b03c6088389c14d49fc18fa0c20bb3d457 +property 96 v2 brotli reference/property/generated-0096-v2.parquet julia/property/generated-0096-v2-brotli.parquet 4e93e938a0d87c73c04887674fb77e38e3cc230bf43b42b2a2f73dbfbdbff095 5625bc4932c5cbbbdac746275fb390418edc70998d8ef8fa92ebddcfafaac59f +property 96 v2 gzip reference/property/generated-0096-v2.parquet julia/property/generated-0096-v2-gzip.parquet 4e93e938a0d87c73c04887674fb77e38e3cc230bf43b42b2a2f73dbfbdbff095 c30b9786dad53e7fb9cef306ae2a37ac72b4e380a9bb69f3e2a677a365cee72b +property 96 v2 lz4_raw reference/property/generated-0096-v2.parquet julia/property/generated-0096-v2-lz4_raw.parquet 4e93e938a0d87c73c04887674fb77e38e3cc230bf43b42b2a2f73dbfbdbff095 95ef9e11914c472e812f74e910ca71923de940239aa3885316b17823b707ccc1 +property 96 v2 snappy reference/property/generated-0096-v2.parquet julia/property/generated-0096-v2-snappy.parquet 4e93e938a0d87c73c04887674fb77e38e3cc230bf43b42b2a2f73dbfbdbff095 a8ab83122115d1b460c15153d3da506b5152e5c2bb16a381f90c8678b2fc5855 +property 96 v2 uncompressed reference/property/generated-0096-v2.parquet julia/property/generated-0096-v2-uncompressed.parquet 4e93e938a0d87c73c04887674fb77e38e3cc230bf43b42b2a2f73dbfbdbff095 4c9510f3fa4086d93a9974d5f53d167420e5da84fd55a305af1c41bf7832a3bd +property 96 v2 zstd reference/property/generated-0096-v2.parquet julia/property/generated-0096-v2-zstd.parquet 4e93e938a0d87c73c04887674fb77e38e3cc230bf43b42b2a2f73dbfbdbff095 6c373d2d0375604bf66bc5bfeabe66d27b671140ac2977b6aa526aaa9cd02775 +property 106 v2 brotli reference/property/generated-0106-v2.parquet julia/property/generated-0106-v2-brotli.parquet 9549669e5fa45926ccb32a251d151b043413f32d8c82a4c916da8b1f4fd91f02 9ea9011689b8e0ec83c60939017c36ce215b973e4e8b995115018e92704d5c45 +property 106 v2 gzip reference/property/generated-0106-v2.parquet julia/property/generated-0106-v2-gzip.parquet 9549669e5fa45926ccb32a251d151b043413f32d8c82a4c916da8b1f4fd91f02 c1af4a2b192f23d2cfe6cbe5c966c729c703c300c7462c40cee244324cbf91b3 +property 106 v2 lz4_raw reference/property/generated-0106-v2.parquet julia/property/generated-0106-v2-lz4_raw.parquet 9549669e5fa45926ccb32a251d151b043413f32d8c82a4c916da8b1f4fd91f02 d31b395d03a6597db1eb57b5c76502e69fcaf4c058d4e33cd6f239ddec6d9264 +property 106 v2 snappy reference/property/generated-0106-v2.parquet julia/property/generated-0106-v2-snappy.parquet 9549669e5fa45926ccb32a251d151b043413f32d8c82a4c916da8b1f4fd91f02 1363b2afc42817100c03a59aff5214fd8abd239c959a8a364c1267b71403501b +property 106 v2 uncompressed reference/property/generated-0106-v2.parquet julia/property/generated-0106-v2-uncompressed.parquet 9549669e5fa45926ccb32a251d151b043413f32d8c82a4c916da8b1f4fd91f02 23927d0148d0471cb403ef071db11942e861f687676eb547f3ed85672e01e36d +property 106 v2 zstd reference/property/generated-0106-v2.parquet julia/property/generated-0106-v2-zstd.parquet 9549669e5fa45926ccb32a251d151b043413f32d8c82a4c916da8b1f4fd91f02 8b45738b2ad1723013575894a3fb854e4747bdf0848fda3a5e26464e62aa4dfb +property 107 v1 brotli reference/property/generated-0107-v1.parquet julia/property/generated-0107-v1-brotli.parquet ed92a45431f261d19369d0e802a8d377f5e0f30d8ade22f3e38d67ea8ffcec04 8b7680549d83e5aaf5fd06529bc11c895c8912bed07f5d768b8a0167a07c666b +property 107 v1 gzip reference/property/generated-0107-v1.parquet julia/property/generated-0107-v1-gzip.parquet ed92a45431f261d19369d0e802a8d377f5e0f30d8ade22f3e38d67ea8ffcec04 5a971be9aff80231b3ca44afbc53b14f50066ac6564f81976ac52490391e9327 +property 107 v1 lz4_raw reference/property/generated-0107-v1.parquet julia/property/generated-0107-v1-lz4_raw.parquet ed92a45431f261d19369d0e802a8d377f5e0f30d8ade22f3e38d67ea8ffcec04 1808e7743750f387737cd11b75dd40afc0555868b82e69dcbb6afa3c9fb7afb8 +property 107 v1 snappy reference/property/generated-0107-v1.parquet julia/property/generated-0107-v1-snappy.parquet ed92a45431f261d19369d0e802a8d377f5e0f30d8ade22f3e38d67ea8ffcec04 f78c20178f34150fc9b7d0640a3b4fea8938f37520896f401daa93e3378bfe0e +property 107 v1 uncompressed reference/property/generated-0107-v1.parquet julia/property/generated-0107-v1-uncompressed.parquet ed92a45431f261d19369d0e802a8d377f5e0f30d8ade22f3e38d67ea8ffcec04 8e739a9d444efd3b8f86936461574533f8712ccc62f558efbaea95a88772603a +property 107 v1 zstd reference/property/generated-0107-v1.parquet julia/property/generated-0107-v1-zstd.parquet ed92a45431f261d19369d0e802a8d377f5e0f30d8ade22f3e38d67ea8ffcec04 dd7666c4c882965d925acbdd6d0b8aec1f55fa50bca3ca7d8a264659cbed466b +property 117 v1 brotli reference/property/generated-0117-v1.parquet julia/property/generated-0117-v1-brotli.parquet 5b099ea68041ecc9326ca32bb384e6b185bd2c77f5dcdb33f5820fcaeec5d152 aa8ced8071e1f522a15c99a824a5b4bb936df197b9a4becb206cdf829ae0f2db +property 117 v1 gzip reference/property/generated-0117-v1.parquet julia/property/generated-0117-v1-gzip.parquet 5b099ea68041ecc9326ca32bb384e6b185bd2c77f5dcdb33f5820fcaeec5d152 d70bbb449a0a2c69ddb2f61b97cff41df2bc5a80712bcde8d61de72a111d9dab +property 117 v1 lz4_raw reference/property/generated-0117-v1.parquet julia/property/generated-0117-v1-lz4_raw.parquet 5b099ea68041ecc9326ca32bb384e6b185bd2c77f5dcdb33f5820fcaeec5d152 07f6401ff94b99450b66f60561110aa25763f9340efafc5748da2e72ce50c76f +property 117 v1 snappy reference/property/generated-0117-v1.parquet julia/property/generated-0117-v1-snappy.parquet 5b099ea68041ecc9326ca32bb384e6b185bd2c77f5dcdb33f5820fcaeec5d152 f43ce5013aea4784cc23f0a39fa17df2e5fc2c505bb4cf9e37d7c1011931667f +property 117 v1 uncompressed reference/property/generated-0117-v1.parquet julia/property/generated-0117-v1-uncompressed.parquet 5b099ea68041ecc9326ca32bb384e6b185bd2c77f5dcdb33f5820fcaeec5d152 74d10b8ca2124925be4e436462d0dc04e41c45c65a332aa04763a746f33e2fac +property 117 v1 zstd reference/property/generated-0117-v1.parquet julia/property/generated-0117-v1-zstd.parquet 5b099ea68041ecc9326ca32bb384e6b185bd2c77f5dcdb33f5820fcaeec5d152 de62e5733770ea09877dab879eff8b3d477d97fc9523174c167b46e81ba15722 +property 122 v2 brotli reference/property/generated-0122-v2.parquet julia/property/generated-0122-v2-brotli.parquet 48e4673ffb51afd736278003f2da71b6b56d113a541c16b2684a36319fa784c9 242307ce4a7d3571df8ad625904e0138c608a325f3f20c71cce953baa265c6fd +property 122 v2 gzip reference/property/generated-0122-v2.parquet julia/property/generated-0122-v2-gzip.parquet 48e4673ffb51afd736278003f2da71b6b56d113a541c16b2684a36319fa784c9 4f9221c2c25e40ef0ec4824a181dac9df68b9e5acd5045d3ee3f3df3e9075c34 +property 122 v2 lz4_raw reference/property/generated-0122-v2.parquet julia/property/generated-0122-v2-lz4_raw.parquet 48e4673ffb51afd736278003f2da71b6b56d113a541c16b2684a36319fa784c9 559532d6a23c0c65a3a7494c071e24a49321adda8edd51cbc12adcd06dfd46af +property 122 v2 snappy reference/property/generated-0122-v2.parquet julia/property/generated-0122-v2-snappy.parquet 48e4673ffb51afd736278003f2da71b6b56d113a541c16b2684a36319fa784c9 fe43353a178efa494fdb6b9985383a91bcbac6f97111e8b9f3c7df355b549068 +property 122 v2 uncompressed reference/property/generated-0122-v2.parquet julia/property/generated-0122-v2-uncompressed.parquet 48e4673ffb51afd736278003f2da71b6b56d113a541c16b2684a36319fa784c9 d083e87b924925b8ad7407841ace216b86abe58d3720cbbe46e93afc6b0763f1 +property 122 v2 zstd reference/property/generated-0122-v2.parquet julia/property/generated-0122-v2-zstd.parquet 48e4673ffb51afd736278003f2da71b6b56d113a541c16b2684a36319fa784c9 ae69b09dcdca2334d1f672843b749a289186c268db7287e81e88354694add9df +property 123 v1 brotli reference/property/generated-0123-v1.parquet julia/property/generated-0123-v1-brotli.parquet bdf51ead923cbebf69df0bb1cdcc340a98b69e3fc7d83baf1b34a932cefa7a35 41a87713e90bdfdfb76a01d28e63f3bb478325b38d22b693182f627119eb33cf +property 123 v1 gzip reference/property/generated-0123-v1.parquet julia/property/generated-0123-v1-gzip.parquet bdf51ead923cbebf69df0bb1cdcc340a98b69e3fc7d83baf1b34a932cefa7a35 c68589c51eee0e3a32794157242399a67795a131e15b3b8cecc78c25b46250c5 +property 123 v1 lz4_raw reference/property/generated-0123-v1.parquet julia/property/generated-0123-v1-lz4_raw.parquet bdf51ead923cbebf69df0bb1cdcc340a98b69e3fc7d83baf1b34a932cefa7a35 e1608b8489b6da60abaece8104fbb9ca489e8a5fc7f11d2157aba706c95d993f +property 123 v1 snappy reference/property/generated-0123-v1.parquet julia/property/generated-0123-v1-snappy.parquet bdf51ead923cbebf69df0bb1cdcc340a98b69e3fc7d83baf1b34a932cefa7a35 79201ccbb8345be91f651c119a77b5f481775144d85784e938532d9d9d01e614 +property 123 v1 uncompressed reference/property/generated-0123-v1.parquet julia/property/generated-0123-v1-uncompressed.parquet bdf51ead923cbebf69df0bb1cdcc340a98b69e3fc7d83baf1b34a932cefa7a35 239a8fad2fc0d36640a9948d15b8fbff9ba0902540bc9437d9a78602f731080c +property 123 v1 zstd reference/property/generated-0123-v1.parquet julia/property/generated-0123-v1-zstd.parquet bdf51ead923cbebf69df0bb1cdcc340a98b69e3fc7d83baf1b34a932cefa7a35 65ba1030bafaeb9f7a603e877ff27e999d28da79728bf94ba124fb4f64bb7f31 +property 144 v2 brotli reference/property/generated-0144-v2.parquet julia/property/generated-0144-v2-brotli.parquet d7310128c56d92463cd54fe3906a9e1b02f0c4815e32e32390b827347fa1f69a b72ca7250bbeec02a714bf3c8b5c13e2dbf30c66264905957314d7a13ba18d16 +property 144 v2 gzip reference/property/generated-0144-v2.parquet julia/property/generated-0144-v2-gzip.parquet d7310128c56d92463cd54fe3906a9e1b02f0c4815e32e32390b827347fa1f69a 62b7e66658cfaa2993474ff4b4331fa897f7879d54817a30ec1f749b894229dc +property 144 v2 lz4_raw reference/property/generated-0144-v2.parquet julia/property/generated-0144-v2-lz4_raw.parquet d7310128c56d92463cd54fe3906a9e1b02f0c4815e32e32390b827347fa1f69a efcaf5caedb62083bc5560af6942dc18271303eb2e9954c7e2b62343d5baf866 +property 144 v2 snappy reference/property/generated-0144-v2.parquet julia/property/generated-0144-v2-snappy.parquet d7310128c56d92463cd54fe3906a9e1b02f0c4815e32e32390b827347fa1f69a f0c068d591f433e8503a111ff75a28889253d5101a679ef31550c6b9cb4e74a4 +property 144 v2 uncompressed reference/property/generated-0144-v2.parquet julia/property/generated-0144-v2-uncompressed.parquet d7310128c56d92463cd54fe3906a9e1b02f0c4815e32e32390b827347fa1f69a 9184b278ecbde6ce8d6adfd4a3a6cafffb836f042ebfacba1a5a560435a6a8e8 +property 144 v2 zstd reference/property/generated-0144-v2.parquet julia/property/generated-0144-v2-zstd.parquet d7310128c56d92463cd54fe3906a9e1b02f0c4815e32e32390b827347fa1f69a 282e2c701498e2958254fc947308d289af2d52b413690ccb3fe35b3292af20ca +property 145 v1 brotli reference/property/generated-0145-v1.parquet julia/property/generated-0145-v1-brotli.parquet 58e975512b1d6266190f59f67c65b7bd0a39637ccecd4e5d01d152765ceefb52 3b8cf3a99f866819fccaac561f70d11e2c0957a21ea761dfd4212b6e3ec5a74a +property 145 v1 gzip reference/property/generated-0145-v1.parquet julia/property/generated-0145-v1-gzip.parquet 58e975512b1d6266190f59f67c65b7bd0a39637ccecd4e5d01d152765ceefb52 a0804e8c338a8995f7ba041ce4bac0b71b785cfdf8082b4e035514ddf2273a11 +property 145 v1 lz4_raw reference/property/generated-0145-v1.parquet julia/property/generated-0145-v1-lz4_raw.parquet 58e975512b1d6266190f59f67c65b7bd0a39637ccecd4e5d01d152765ceefb52 b12a7976be84f15d9fe1d03da40d95373719438d4c8c32deb9766ab70ec68f5d +property 145 v1 snappy reference/property/generated-0145-v1.parquet julia/property/generated-0145-v1-snappy.parquet 58e975512b1d6266190f59f67c65b7bd0a39637ccecd4e5d01d152765ceefb52 4b952c788b01bbd257f544c92a7a6639bdb88e4634c4422f1aad66ff57a1718c +property 145 v1 uncompressed reference/property/generated-0145-v1.parquet julia/property/generated-0145-v1-uncompressed.parquet 58e975512b1d6266190f59f67c65b7bd0a39637ccecd4e5d01d152765ceefb52 2a62d573ae8c6a385e5d89e8437c6e5b0c4cc8c1c50e581b5ee2b764a7e6b94a +property 145 v1 zstd reference/property/generated-0145-v1.parquet julia/property/generated-0145-v1-zstd.parquet 58e975512b1d6266190f59f67c65b7bd0a39637ccecd4e5d01d152765ceefb52 c2ee4c58f5a4ca9ad53f6e59b2b0c3bfdc4b6a99d845e8cc211f71e4cab4aa3e +property 148 v2 brotli reference/property/generated-0148-v2.parquet julia/property/generated-0148-v2-brotli.parquet 3dcb776193896ccc5010d32b869b471470d5c0fae48c7e639647856d58f54774 3dd43312ed90e625120600d9f6cec2b8296c7c430208aff393083a1cc5897446 +property 148 v2 gzip reference/property/generated-0148-v2.parquet julia/property/generated-0148-v2-gzip.parquet 3dcb776193896ccc5010d32b869b471470d5c0fae48c7e639647856d58f54774 2417e12ceb6b45d359eb81d6ee1b43586f6bfb52c8f8ffb28485a245e139cc46 +property 148 v2 lz4_raw reference/property/generated-0148-v2.parquet julia/property/generated-0148-v2-lz4_raw.parquet 3dcb776193896ccc5010d32b869b471470d5c0fae48c7e639647856d58f54774 fce64d41be8e6b8851743eed8b12a1236d198fd85b32054195ebdbcfc97eb463 +property 148 v2 snappy reference/property/generated-0148-v2.parquet julia/property/generated-0148-v2-snappy.parquet 3dcb776193896ccc5010d32b869b471470d5c0fae48c7e639647856d58f54774 e9090565416005e2525656591e5fb4546a9705d854d5a5f11173f083f3f5417f +property 148 v2 uncompressed reference/property/generated-0148-v2.parquet julia/property/generated-0148-v2-uncompressed.parquet 3dcb776193896ccc5010d32b869b471470d5c0fae48c7e639647856d58f54774 18849f8a7f133c8b3c4916a9afcdd13f3a1b1a83bc275d47d0fa7efdd23da3f2 +property 148 v2 zstd reference/property/generated-0148-v2.parquet julia/property/generated-0148-v2-zstd.parquet 3dcb776193896ccc5010d32b869b471470d5c0fae48c7e639647856d58f54774 d8cc54dc2a8d467221e227063b4232d87fcf036942f8829e90ca7214a087f64c +property 178 v1 brotli reference/property/generated-0178-v1.parquet julia/property/generated-0178-v1-brotli.parquet 00b731d38b4edf95178eec63f7b0abbf9b717417f7a7edd42f5778e7056adf09 3482317e88afe245a159e8dba3468e614b238097a81fb7d21bc17a825b8860da +property 178 v1 gzip reference/property/generated-0178-v1.parquet julia/property/generated-0178-v1-gzip.parquet 00b731d38b4edf95178eec63f7b0abbf9b717417f7a7edd42f5778e7056adf09 51af92f6754c830aa89ae00d730f09b712535eedd7787f5d323a35e843365fb9 +property 178 v1 lz4_raw reference/property/generated-0178-v1.parquet julia/property/generated-0178-v1-lz4_raw.parquet 00b731d38b4edf95178eec63f7b0abbf9b717417f7a7edd42f5778e7056adf09 845b45a555d839b647615c683f7b021cba6e302803331566aa5254b939dfd87c +property 178 v1 snappy reference/property/generated-0178-v1.parquet julia/property/generated-0178-v1-snappy.parquet 00b731d38b4edf95178eec63f7b0abbf9b717417f7a7edd42f5778e7056adf09 4940ffda40a210679fe6000ff2c19c91b548ac1a5c3a5ef203a12583384974d4 +property 178 v1 uncompressed reference/property/generated-0178-v1.parquet julia/property/generated-0178-v1-uncompressed.parquet 00b731d38b4edf95178eec63f7b0abbf9b717417f7a7edd42f5778e7056adf09 897baaed11dde7e8c9172a666fd9acdce838bf8c16e54a24a5b9786ed0d8524e +property 178 v1 zstd reference/property/generated-0178-v1.parquet julia/property/generated-0178-v1-zstd.parquet 00b731d38b4edf95178eec63f7b0abbf9b717417f7a7edd42f5778e7056adf09 cc39cd58f6d4e122f1f1332671566a45dfe62f0eb9519880995afe6618122901 +property 191 v2 brotli reference/property/generated-0191-v2.parquet julia/property/generated-0191-v2-brotli.parquet 565be1c5e017366f1504bf01007ac32418b20bf6113bd57b2da8d4c2c9ec1861 d09dd44d135b8eb348a3ff54f0257fa6dbc0648c08b8ffa09e76c03f06b54086 +property 191 v2 gzip reference/property/generated-0191-v2.parquet julia/property/generated-0191-v2-gzip.parquet 565be1c5e017366f1504bf01007ac32418b20bf6113bd57b2da8d4c2c9ec1861 dfcb1e27fb957ca45329ff661b5baa92200d18c0948935e3bffc67f81518e320 +property 191 v2 lz4_raw reference/property/generated-0191-v2.parquet julia/property/generated-0191-v2-lz4_raw.parquet 565be1c5e017366f1504bf01007ac32418b20bf6113bd57b2da8d4c2c9ec1861 2afe65db767440a3b8ccb85701425b91998cbfef451ece24faf35f5b3ca0bb99 +property 191 v2 snappy reference/property/generated-0191-v2.parquet julia/property/generated-0191-v2-snappy.parquet 565be1c5e017366f1504bf01007ac32418b20bf6113bd57b2da8d4c2c9ec1861 6d3940784ce7ac1fa1bdef55efa6da1a6613024bdc94427759b13ef4813de528 +property 191 v2 uncompressed reference/property/generated-0191-v2.parquet julia/property/generated-0191-v2-uncompressed.parquet 565be1c5e017366f1504bf01007ac32418b20bf6113bd57b2da8d4c2c9ec1861 914d6869b729367dab753d8caaecd2171c9d466d317bb1edc052da970e54834c +property 191 v2 zstd reference/property/generated-0191-v2.parquet julia/property/generated-0191-v2-zstd.parquet 565be1c5e017366f1504bf01007ac32418b20bf6113bd57b2da8d4c2c9ec1861 7f0a930b1365398fb595551eea265238b9b4d320011d7f6e392958056d09b077 +property 202 v2 brotli reference/property/generated-0202-v2.parquet julia/property/generated-0202-v2-brotli.parquet 0fdef6d891b1731eaf217c63e336322f98aa29ced51a8b645335ace0bef2a5b4 3025ed8d78da6ad0638f99e535ffdb66b5fa73dcb3e3e32762042ed1f4f8085f +property 202 v2 gzip reference/property/generated-0202-v2.parquet julia/property/generated-0202-v2-gzip.parquet 0fdef6d891b1731eaf217c63e336322f98aa29ced51a8b645335ace0bef2a5b4 7f97caf5318185ca59cd1d11ab0b8ad6c18548b1630de151402885f9c3c0aaa2 +property 202 v2 lz4_raw reference/property/generated-0202-v2.parquet julia/property/generated-0202-v2-lz4_raw.parquet 0fdef6d891b1731eaf217c63e336322f98aa29ced51a8b645335ace0bef2a5b4 a601f114bd71eaa371cb47134063c7c498c804de00326b8aaa0edd215c34d768 +property 202 v2 snappy reference/property/generated-0202-v2.parquet julia/property/generated-0202-v2-snappy.parquet 0fdef6d891b1731eaf217c63e336322f98aa29ced51a8b645335ace0bef2a5b4 aadaaa13a90872df62258193e9dbdc211c1bd1f8795517c056823cc9d7fac7ee +property 202 v2 uncompressed reference/property/generated-0202-v2.parquet julia/property/generated-0202-v2-uncompressed.parquet 0fdef6d891b1731eaf217c63e336322f98aa29ced51a8b645335ace0bef2a5b4 3d37eac2ddb0d4cf3dfaa30d1d3d2b93927073886a387c3a67d00eecc48ac76b +property 202 v2 zstd reference/property/generated-0202-v2.parquet julia/property/generated-0202-v2-zstd.parquet 0fdef6d891b1731eaf217c63e336322f98aa29ced51a8b645335ace0bef2a5b4 bf4a0f6ee136b1a6b706873ee8c2f6c77758be4e8f5bb630f20e19342167137c +property 249 v1 brotli reference/property/generated-0249-v1.parquet julia/property/generated-0249-v1-brotli.parquet c76283d134e73f6e7e74dcf868d79876973027280b66c9505a4b026f58c1f076 f37a9461c080ae22bb2772096f375b15579af426079e78a1ad9d914a3efda186 +property 249 v1 gzip reference/property/generated-0249-v1.parquet julia/property/generated-0249-v1-gzip.parquet c76283d134e73f6e7e74dcf868d79876973027280b66c9505a4b026f58c1f076 f85151062445dc7a43bd4d873c582e5a5a40fa0fbc3870193ee0ef215249b474 +property 249 v1 lz4_raw reference/property/generated-0249-v1.parquet julia/property/generated-0249-v1-lz4_raw.parquet c76283d134e73f6e7e74dcf868d79876973027280b66c9505a4b026f58c1f076 a1590c1b3dda367851b585e3c470ce2447635a38527cd483015654e8240b6434 +property 249 v1 snappy reference/property/generated-0249-v1.parquet julia/property/generated-0249-v1-snappy.parquet c76283d134e73f6e7e74dcf868d79876973027280b66c9505a4b026f58c1f076 9a779936dc5e17570b030d67020b07e35325b01a401ca07f9ba80a16320932a6 +property 249 v1 uncompressed reference/property/generated-0249-v1.parquet julia/property/generated-0249-v1-uncompressed.parquet c76283d134e73f6e7e74dcf868d79876973027280b66c9505a4b026f58c1f076 6dbac686a7041c3bcf82b9fef229557ef720ea1e5c23a6cd9e919e81a0e609d8 +property 249 v1 zstd reference/property/generated-0249-v1.parquet julia/property/generated-0249-v1-zstd.parquet c76283d134e73f6e7e74dcf868d79876973027280b66c9505a4b026f58c1f076 13ed8aacb78ffdfb57289ad4c860b87fea66922845f093d9570a35d604ca4bed +property 251 v1 brotli reference/property/generated-0251-v1.parquet julia/property/generated-0251-v1-brotli.parquet 4e953527c320c16864803ca3d9b0063437d113c15be4bc5b458b2fdb8b49ba48 e5b7455926b110110f87ba49321a2553f210f79ca76f4bee1aa653b2f62e061c +property 251 v1 gzip reference/property/generated-0251-v1.parquet julia/property/generated-0251-v1-gzip.parquet 4e953527c320c16864803ca3d9b0063437d113c15be4bc5b458b2fdb8b49ba48 84062ae72e4c44bdf10b7c45b9d016df6660ff55c87f1a6bf9305cd4687fa850 +property 251 v1 lz4_raw reference/property/generated-0251-v1.parquet julia/property/generated-0251-v1-lz4_raw.parquet 4e953527c320c16864803ca3d9b0063437d113c15be4bc5b458b2fdb8b49ba48 59e0ba63b19a2b1f5f6659342a7d2b5fcace31d3064f8ab86261d6459a6ac8a4 +property 251 v1 snappy reference/property/generated-0251-v1.parquet julia/property/generated-0251-v1-snappy.parquet 4e953527c320c16864803ca3d9b0063437d113c15be4bc5b458b2fdb8b49ba48 d27e85b23d4c4c8477584a33d2ef805dff5a034769f1a686aae1b2cc91b9fe1f +property 251 v1 uncompressed reference/property/generated-0251-v1.parquet julia/property/generated-0251-v1-uncompressed.parquet 4e953527c320c16864803ca3d9b0063437d113c15be4bc5b458b2fdb8b49ba48 aca5729d1c9431776b01d2d948b0854deffa6dee6c6784a74acb885f1e85d6cc +property 251 v1 zstd reference/property/generated-0251-v1.parquet julia/property/generated-0251-v1-zstd.parquet 4e953527c320c16864803ca3d9b0063437d113c15be4bc5b458b2fdb8b49ba48 a5935c6e05ff0697aed875b7bf6db089ae1b966f1a66e2e4d3e4d758b8429965 +external arrow-rs-duplicate-keys v1 uncompressed reference/external/arrow-rs/arrow-rs-duplicate-keys_v1.parquet julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v1.parquet acf5536a16167a2a870dfd55ad2a5d208712069d8e0f034cf396fa6d6c193184 6d0b7ea1d1c066a8226ec78d7cfbf47232cbce5f0895267fd6033671ae620e82 +external arrow-rs-duplicate-keys v2 uncompressed reference/external/arrow-rs/arrow-rs-duplicate-keys_v2.parquet julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v2.parquet baaf9a09bc397db8c0b1d0453fa2d87e36f6a74bbc56778be39d27304007cbb3 aa1bc9a936c93d5fa8db4267d0cd45fb8c564c892b362aff120651d479e1e56a +external arrow-rs-list-rule3-unannotated-near-neighbor v1 uncompressed reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet 40eb4521da12a9bd5db38cfe9e1f31dd91bd1095ec5b24b94f9a02318e40128f 5fbad40b085b1e68911907b498f07f580abc70fa043adc342de939b6d1441485 +external arrow-rs-list-rule3-unannotated-near-neighbor v2 uncompressed reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet 4fe0319ea46248612489d845eb6fdbfa127077f06a9e7a2efc14fa67def66ce1 81a6be40524b8efdc9d5fc0b8c58e0e558506cffe7c08918e217d259a723637a +external arrow-rs-list-rule3 v1 uncompressed reference/external/arrow-rs/arrow-rs-list-rule3_v1.parquet julia/rewrite/arrow-rs/arrow-rs-list-rule3_v1.parquet 916475c20dde6afe338d60e43d2e4a6293c2bc26efb6dd439d1d7d9299834bc6 aaf0b99a25f0fde7a7d054ad5a562af1f080a84ed8d750ce5ad9741c00bfcb9a +external arrow-rs-list-rule3 v2 uncompressed reference/external/arrow-rs/arrow-rs-list-rule3_v2.parquet julia/rewrite/arrow-rs/arrow-rs-list-rule3_v2.parquet ef72761c509f8bc302986d130f6d7d9382b975c8a3462a955c599bd5f427edac 8332c983b137e04d84bec5a7d46f3f5b8270de65db2e3062ab87896b4c7ef744 +external arrow-rs-optional-key-present v1 uncompressed reference/external/arrow-rs/arrow-rs-optional-key-present_v1.parquet julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v1.parquet a33be66fe1a5e7189c457b49163e5a4b658228ffb5054a40cd22f20f731ba21a c32444882e1fa633aa3411449d417d904afb70337907c80503f998dfe0a943cd +external arrow-rs-optional-key-present v2 uncompressed reference/external/arrow-rs/arrow-rs-optional-key-present_v2.parquet julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v2.parquet f010b919bcc2d4e8c42f6d97df651266de06b6352f6b993756fd0a007e1c5028 30e3bea2ba646ab7bf0a45536904425e6f41478beddd60fbfa72eb1c6695ecac +external list_direct_map v1 uncompressed reference/external/parquet-java/list_direct_map.v1.parquet julia/rewrite/parquet-java/list_direct_map.v1.parquet c84b4c582a00245acbc6330c80a3437f0007e1c2311adf31025b6952a95999e3 19cc449cde41bac260b57b46b425a8225e456274915580eafb51182b86b8eded +external list_direct_map v2 uncompressed reference/external/parquet-java/list_direct_map.v2.parquet julia/rewrite/parquet-java/list_direct_map.v2.parquet c2d8ac178692160961f93b2a572a0bd4b19f71c63eddb56f7585c3e5a30c13eb b4f7a5818830bbfa3a3a8fe935fcfa645889e9a2ac9f3615e6da8b98c36e4595 +external list_direct_map_utf8 v1 uncompressed reference/external/parquet-java/list_direct_map_utf8.v1.parquet julia/rewrite/parquet-java/list_direct_map_utf8.v1.parquet 4f7faf3e2bef6c043d15e88feeb796728df51a9648abb81a301e0f6e6f97fd0b df6e405e214e651df86034c4b67b0720a3d71d8eccfcff1b75ce0c8397a4bd8a +external list_direct_map_utf8 v2 uncompressed reference/external/parquet-java/list_direct_map_utf8.v2.parquet julia/rewrite/parquet-java/list_direct_map_utf8.v2.parquet edc66d338474404c9c7db8a4b93d5f1def987b4883f5aefe7d042c348aa282de 7bcbd22ad1305d5d2d67ed0143f8c150458951f4a1a08d9df446f23424768180 +external list_rule1_primitive v1 uncompressed reference/external/parquet-java/list_rule1_primitive.v1.parquet julia/rewrite/parquet-java/list_rule1_primitive.v1.parquet 3466fa00347fae5832a8b751cd50d74e0af0df490d7262bdccf36f0c2f38c2db ac53fc3e511a056e0a466ce5e2b8be4c79d73e3871dd2f397155aae3e59f7961 +external list_rule1_primitive v2 uncompressed reference/external/parquet-java/list_rule1_primitive.v2.parquet julia/rewrite/parquet-java/list_rule1_primitive.v2.parquet 056989f7d112e735fc4f4e835278ba58610e522e1aa3a3e05afdaf540c0463cd 6e67250b399d2cc718478d01e9343866d188cd3663b7cfa4553e340266342670 +external list_rule2_struct v1 uncompressed reference/external/parquet-java/list_rule2_struct.v1.parquet julia/rewrite/parquet-java/list_rule2_struct.v1.parquet b3c6ddd69b4b75bfe0ba1fcb21aca6db5d7b7c083fda0cd106b7229baa0a2f7b d1c1706a5bd289ae64ef34db86e3fd48fe56f1d67788736b050aa679ab3f4a67 +external list_rule2_struct v2 uncompressed reference/external/parquet-java/list_rule2_struct.v2.parquet julia/rewrite/parquet-java/list_rule2_struct.v2.parquet 2b059b3b5b375c736e7d0abb69a5f1e8dc48606008ffd9e3910f34d3543fc6a6 67fc6cc8769e2baa9a86d5642794ca38b274eca4649fd67e50525cf51e4e6ecf +external list_rule3_nested v1 uncompressed reference/external/parquet-java/list_rule3_nested.v1.parquet julia/rewrite/parquet-java/list_rule3_nested.v1.parquet c85821a7f6593ad3092f2efd3c8acc602f4969eb8dc36d49f1868b22f1d7d56a 847533a54c2687dd0eb849ac5e67e52dfd21e936eb9c65c5ff8bfa37e85ce87b +external list_rule3_nested v2 uncompressed reference/external/parquet-java/list_rule3_nested.v2.parquet julia/rewrite/parquet-java/list_rule3_nested.v2.parquet 5333c37c239ae57781c63eec972537eb428e275da8d3cca0634acb963989e73e 6c5cff739e8349b593bd4f96fa8be3c4cf4a0fbcd8750776f42b08c100906323 +external list_rule3_unannotated_diagnostic v1 uncompressed reference/external/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet ab2c1a962ee992845540c6aa04056f15cdc4b40cf89627fb5160dd3fb07a5123 c0cbc94cd2e842a3f997e163d61f002203522064a184707ea7bbaeef305e1a05 +external list_rule3_unannotated_diagnostic v2 uncompressed reference/external/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet 51ece6eccb1123e817c7e4a9f5f0ff235299501c7e7aa4e78f2878ecb4ae4e5f 542231a9bc8d1cf49008eecc1c0fb7a4b94738dea837e7714b4e81b8f3ea24ac +external list_rule4_array v1 uncompressed reference/external/parquet-java/list_rule4_array.v1.parquet julia/rewrite/parquet-java/list_rule4_array.v1.parquet 4b7a1f524f1f04a6fcb2f8ebabf1f82cf33d709cc154207e43e95ab61d09ed1a 22e64d7cff4cb34d9bc980cb1380c2174ad6dda164ef26ce83985e2c2ab21e80 +external list_rule4_array v2 uncompressed reference/external/parquet-java/list_rule4_array.v2.parquet julia/rewrite/parquet-java/list_rule4_array.v2.parquet 966d39ae1c7d1c011b3e2405ef0be9ff9d23372516abcc2060331b61c00bc642 d8ef8f67c22b8a33d92881055bce7592b312cb59f1fbc1f9af2d352eb0b4728c +external list_rule4_tuple v1 uncompressed reference/external/parquet-java/list_rule4_tuple.v1.parquet julia/rewrite/parquet-java/list_rule4_tuple.v1.parquet 2b7e4ae67a92182f44cff1c584096bc6b6d741106fe56eb20b0e2d3a41d0a170 b095cdd84f421ad1308f2aefd22e4d4e35fb1a3519d33c4eec74eee7a37ed5d8 +external list_rule4_tuple v2 uncompressed reference/external/parquet-java/list_rule4_tuple.v2.parquet julia/rewrite/parquet-java/list_rule4_tuple.v2.parquet 9d51d23da84416b0b520e0fdb80796d80923b5c8cbdeeea454a8c422a897934a 0c6f0006fbf1f8c0662bbabaa4c9c977f3c9ecf58fa7753d5c3a7777141d635e +external list_rule5_optional_extended v1 uncompressed reference/external/parquet-java/list_rule5_optional_extended.v1.parquet julia/rewrite/parquet-java/list_rule5_optional_extended.v1.parquet 5281d05e2317fbc30e280c18b15078d0676b5ef22b1ee56d9788dc2e45d45af0 831d4d653dfe80f07be8abb12cf996414e781df0df5a1e0870c487de0eda1507 +external list_rule5_optional_extended v2 uncompressed reference/external/parquet-java/list_rule5_optional_extended.v2.parquet julia/rewrite/parquet-java/list_rule5_optional_extended.v2.parquet e2b4bca7e67bac4dafdca1a4aea01d3bdf764a6ece37419edd2c39699efec6c4 b9c556b6052ca645a65f952635fa3b0d0323848ddffeae4765e7da9f642a30c3 +external list_rule5_optional_paired v1 uncompressed reference/external/parquet-java/list_rule5_optional_paired.v1.parquet julia/rewrite/parquet-java/list_rule5_optional_paired.v1.parquet 2873a69f47bd82a93625a78fda96a58a5783c5f473f0ace861216df876c9a725 f832162b8ffb40c4e2ab637367b67a606932b1665927f36ec6d33dfd4ea7e0fe +external list_rule5_optional_paired v2 uncompressed reference/external/parquet-java/list_rule5_optional_paired.v2.parquet julia/rewrite/parquet-java/list_rule5_optional_paired.v2.parquet a465d3f92575429b545d4edad4aece7dd30019860ee71af75f61d035c0b00f04 0802e8871fdc62b16c96ab1bfd6cc5ecb68c5186c8c556a893e7c81353a3e06e +external list_rule5_required v1 uncompressed reference/external/parquet-java/list_rule5_required.v1.parquet julia/rewrite/parquet-java/list_rule5_required.v1.parquet e16a41a44e974c44f8ca83021ecd55e016018bab7f86a930b2560af7d66de762 32e8866097d622296b9130e9f53620241e848282372ca977238585d2f4addb3e +external list_rule5_required v2 uncompressed reference/external/parquet-java/list_rule5_required.v2.parquet julia/rewrite/parquet-java/list_rule5_required.v2.parquet a730dbdfca94503ce0f1e6717f96c4636378b2b7f56d593bebae86fa86513127 78ea66349d545dd653f18c59fc9c35630296ca0876bb46705e22c49688da0c79 +external map_arbitrary_names v1 uncompressed reference/external/parquet-java/map_arbitrary_names.v1.parquet julia/rewrite/parquet-java/map_arbitrary_names.v1.parquet 0a6aadfd449407c0e6036d6b894e4f73c9d6a9b7a64b35eba16da2a685954ccb 557584fabf79da29ac15d9eed78c875fa004557622e70db4532d8e92ce8854b5 +external map_arbitrary_names v2 uncompressed reference/external/parquet-java/map_arbitrary_names.v2.parquet julia/rewrite/parquet-java/map_arbitrary_names.v2.parquet b69c7d382417e2be0834bbbc060a5a65548867c659aec51283b77650bccd8de4 f4367ba8815f9b1eecd6b623ff769e6d90008e1597337dfc966dd1a45bced24b +external map_key_only v1 uncompressed reference/external/parquet-java/map_key_only.v1.parquet julia/rewrite/parquet-java/map_key_only.v1.parquet e05e566da94fcdd96d5ca17c4f8f4403541a50c94002bef15639df889fb79102 2eed41707a21af87a89153ddce92d172ca438dd0ad3dd0f9e0ad23c8719b7f0f +external map_key_only v2 uncompressed reference/external/parquet-java/map_key_only.v2.parquet julia/rewrite/parquet-java/map_key_only.v2.parquet 3b6d03072c1046e000097bad8392319678256610a1ad56e7f6611c390cb0a5ef 243697559953c4c33790c2c6e03e24b57a8ed542fbc0b564bb885ad17721ac33 +external map_standalone_mkv v1 uncompressed reference/external/parquet-java/map_standalone_mkv.v1.parquet julia/rewrite/parquet-java/map_standalone_mkv.v1.parquet a99a78cc84bdc17a30d0ab73c28c5edf6a7640b85ceab9470fedaa7369a5f401 96e8b27d829b8fabf7a381beefe6d14ece00b2652ae5bcaccb2238af87a4e872 +external map_standalone_mkv v2 uncompressed reference/external/parquet-java/map_standalone_mkv.v2.parquet julia/rewrite/parquet-java/map_standalone_mkv.v2.parquet 467cdc8abf6514d94aa8698d7b9bcad654684bd5d465cc0b973329a3c9db1f30 ea438ff2e20affac9ebd71d7ce1d1f4989fdde3e7d2d06477932a4fa3c5911ff +external map_standard v1 uncompressed reference/external/parquet-java/map_standard.v1.parquet julia/rewrite/parquet-java/map_standard.v1.parquet 0acccffff97f7ff6867daaa35f57ad1a9609ed3407e81921a275f9c3f40d20d5 9de1ce6ad91d4ef7a29ffd4fc59e60b5ed6aa1fc7781ecd55c806002cdec93d3 +external map_standard v2 uncompressed reference/external/parquet-java/map_standard.v2.parquet julia/rewrite/parquet-java/map_standard.v2.parquet 136bf8f4949fa97f52b337b7a4c349d6fb2ca2f38e2df815adadea5df134c9fd e35ffcc7e60c2ab700724b57466e7daca3ca843e916fd2ba07ebabf6dd055e2e diff --git a/test/conformance/n5/julia-fixtures/julia/model/direct-list-of-map.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/direct-list-of-map.v1.parquet new file mode 100644 index 0000000..777565d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/direct-list-of-map.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/direct-list-of-map.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/direct-list-of-map.v2.parquet new file mode 100644 index 0000000..ea2455b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/direct-list-of-map.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-1.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-1.v1.parquet new file mode 100644 index 0000000..42aca0a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-1.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-1.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-1.v2.parquet new file mode 100644 index 0000000..d0adaf1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-1.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-2.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-2.v1.parquet new file mode 100644 index 0000000..699b63c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-2.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-2.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-2.v2.parquet new file mode 100644 index 0000000..f59aec0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-2.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-3.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-3.v1.parquet new file mode 100644 index 0000000..0b8b67f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-3.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-3.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-3.v2.parquet new file mode 100644 index 0000000..8c74a9b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-3.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-array.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-array.v1.parquet new file mode 100644 index 0000000..2fbc6c2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-array.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-array.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-array.v2.parquet new file mode 100644 index 0000000..ac23475 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-array.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-tuple.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-tuple.v1.parquet new file mode 100644 index 0000000..180075f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-tuple.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-tuple.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-tuple.v2.parquet new file mode 100644 index 0000000..3cf4a96 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-4-tuple.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-extended.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-extended.v1.parquet new file mode 100644 index 0000000..67d5d62 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-extended.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-extended.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-extended.v2.parquet new file mode 100644 index 0000000..13c5504 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-extended.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-paired.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-paired.v1.parquet new file mode 100644 index 0000000..ff6956f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-paired.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-paired.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-paired.v2.parquet new file mode 100644 index 0000000..3ffd07c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-paired.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-required.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-required.v1.parquet new file mode 100644 index 0000000..e2b2dac Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-required.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-required.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-required.v2.parquet new file mode 100644 index 0000000..8f873fc Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/list-rule-5-required.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/map-key-only.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/map-key-only.v1.parquet new file mode 100644 index 0000000..b6a1198 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/map-key-only.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/map-key-only.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/map-key-only.v2.parquet new file mode 100644 index 0000000..b4d0126 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/map-key-only.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/map-optional-key.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/map-optional-key.v1.parquet new file mode 100644 index 0000000..0e8dee4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/map-optional-key.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/map-optional-key.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/map-optional-key.v2.parquet new file mode 100644 index 0000000..57df321 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/map-optional-key.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/map-standard.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/map-standard.v1.parquet new file mode 100644 index 0000000..23ba7fc Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/map-standard.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/map-standard.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/map-standard.v2.parquet new file mode 100644 index 0000000..820ae73 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/map-standard.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/schema-provenance.v1.parquet b/test/conformance/n5/julia-fixtures/julia/model/schema-provenance.v1.parquet new file mode 100644 index 0000000..1caf3b5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/schema-provenance.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/model/schema-provenance.v2.parquet b/test/conformance/n5/julia-fixtures/julia/model/schema-provenance.v2.parquet new file mode 100644 index 0000000..818e905 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/model/schema-provenance.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-brotli.parquet new file mode 100644 index 0000000..f77b99f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-gzip.parquet new file mode 100644 index 0000000..59a39a0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-lz4_raw.parquet new file mode 100644 index 0000000..5450385 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-snappy.parquet new file mode 100644 index 0000000..8873df9 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-uncompressed.parquet new file mode 100644 index 0000000..18d4dc5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-zstd.parquet new file mode 100644 index 0000000..0a30b44 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0002-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-brotli.parquet new file mode 100644 index 0000000..fec5d91 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-gzip.parquet new file mode 100644 index 0000000..9e15d95 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-lz4_raw.parquet new file mode 100644 index 0000000..0d10491 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-snappy.parquet new file mode 100644 index 0000000..2824dc8 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-uncompressed.parquet new file mode 100644 index 0000000..3c52f95 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-zstd.parquet new file mode 100644 index 0000000..6f9c1ed Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0004-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-brotli.parquet new file mode 100644 index 0000000..e6c5faf Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-gzip.parquet new file mode 100644 index 0000000..16373bc Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-lz4_raw.parquet new file mode 100644 index 0000000..2e6ec8f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-snappy.parquet new file mode 100644 index 0000000..94c4496 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-uncompressed.parquet new file mode 100644 index 0000000..24e17f3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-zstd.parquet new file mode 100644 index 0000000..6b42ca8 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0005-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-brotli.parquet new file mode 100644 index 0000000..61b4e7e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-gzip.parquet new file mode 100644 index 0000000..fcf95f6 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-lz4_raw.parquet new file mode 100644 index 0000000..385a273 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-snappy.parquet new file mode 100644 index 0000000..66735e7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-uncompressed.parquet new file mode 100644 index 0000000..37d432b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-zstd.parquet new file mode 100644 index 0000000..0724663 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0006-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-brotli.parquet new file mode 100644 index 0000000..08df1dd Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-gzip.parquet new file mode 100644 index 0000000..01d90f1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-lz4_raw.parquet new file mode 100644 index 0000000..6c66254 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-snappy.parquet new file mode 100644 index 0000000..2398bac Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-uncompressed.parquet new file mode 100644 index 0000000..d3c2ecd Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-zstd.parquet new file mode 100644 index 0000000..9eaa8d2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0007-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-brotli.parquet new file mode 100644 index 0000000..569da25 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-gzip.parquet new file mode 100644 index 0000000..0c2c6a2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-lz4_raw.parquet new file mode 100644 index 0000000..8fc504c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-snappy.parquet new file mode 100644 index 0000000..e6b6fb8 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-uncompressed.parquet new file mode 100644 index 0000000..225b844 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-zstd.parquet new file mode 100644 index 0000000..07bfa10 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0008-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-brotli.parquet new file mode 100644 index 0000000..01a2320 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-gzip.parquet new file mode 100644 index 0000000..15553cf Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-lz4_raw.parquet new file mode 100644 index 0000000..a3b174b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-snappy.parquet new file mode 100644 index 0000000..7d3a73e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-uncompressed.parquet new file mode 100644 index 0000000..c355aef Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-zstd.parquet new file mode 100644 index 0000000..c93b12b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0009-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-brotli.parquet new file mode 100644 index 0000000..2e98f5a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-gzip.parquet new file mode 100644 index 0000000..dfbcd51 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-lz4_raw.parquet new file mode 100644 index 0000000..ffafe6c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-snappy.parquet new file mode 100644 index 0000000..51f197d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-uncompressed.parquet new file mode 100644 index 0000000..25388d2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-zstd.parquet new file mode 100644 index 0000000..1b9e5e3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0010-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-brotli.parquet new file mode 100644 index 0000000..f643cc2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-gzip.parquet new file mode 100644 index 0000000..3cea5f3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-lz4_raw.parquet new file mode 100644 index 0000000..f79be3b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-snappy.parquet new file mode 100644 index 0000000..889026e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-uncompressed.parquet new file mode 100644 index 0000000..ec69fa3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-zstd.parquet new file mode 100644 index 0000000..2a0d9ce Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0011-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-brotli.parquet new file mode 100644 index 0000000..993daee Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-gzip.parquet new file mode 100644 index 0000000..de288fa Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-lz4_raw.parquet new file mode 100644 index 0000000..362b589 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-snappy.parquet new file mode 100644 index 0000000..4841b55 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-uncompressed.parquet new file mode 100644 index 0000000..5603be5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-zstd.parquet new file mode 100644 index 0000000..c8a1447 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0014-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-brotli.parquet new file mode 100644 index 0000000..dda1257 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-gzip.parquet new file mode 100644 index 0000000..f3315ef Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-lz4_raw.parquet new file mode 100644 index 0000000..109180d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-snappy.parquet new file mode 100644 index 0000000..b9b1b14 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-uncompressed.parquet new file mode 100644 index 0000000..6fda6c9 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-zstd.parquet new file mode 100644 index 0000000..e144ca0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0015-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-brotli.parquet new file mode 100644 index 0000000..e354ab5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-gzip.parquet new file mode 100644 index 0000000..e83b746 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-lz4_raw.parquet new file mode 100644 index 0000000..197c4e4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-snappy.parquet new file mode 100644 index 0000000..fcd9c9a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-uncompressed.parquet new file mode 100644 index 0000000..c8f23a3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-zstd.parquet new file mode 100644 index 0000000..ed4d560 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0018-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-brotli.parquet new file mode 100644 index 0000000..d196fb5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-gzip.parquet new file mode 100644 index 0000000..0c1bf94 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-lz4_raw.parquet new file mode 100644 index 0000000..25a86cf Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-snappy.parquet new file mode 100644 index 0000000..b6b019e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-uncompressed.parquet new file mode 100644 index 0000000..a4184bf Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-zstd.parquet new file mode 100644 index 0000000..9800d44 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0028-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-brotli.parquet new file mode 100644 index 0000000..79bd06e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-gzip.parquet new file mode 100644 index 0000000..094e340 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-lz4_raw.parquet new file mode 100644 index 0000000..2ffad2b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-snappy.parquet new file mode 100644 index 0000000..11eefa4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-uncompressed.parquet new file mode 100644 index 0000000..62ef7c9 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-zstd.parquet new file mode 100644 index 0000000..171f44d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0034-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-brotli.parquet new file mode 100644 index 0000000..e6ac04d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-gzip.parquet new file mode 100644 index 0000000..56b4087 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-lz4_raw.parquet new file mode 100644 index 0000000..99a3695 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-snappy.parquet new file mode 100644 index 0000000..6de70a3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-uncompressed.parquet new file mode 100644 index 0000000..d15f776 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-zstd.parquet new file mode 100644 index 0000000..d44c862 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0038-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-brotli.parquet new file mode 100644 index 0000000..97aa86a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-gzip.parquet new file mode 100644 index 0000000..f11f2ce Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-lz4_raw.parquet new file mode 100644 index 0000000..fb6d66c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-snappy.parquet new file mode 100644 index 0000000..a30a31c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-uncompressed.parquet new file mode 100644 index 0000000..80cdab4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-zstd.parquet new file mode 100644 index 0000000..6ae8099 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0048-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-brotli.parquet new file mode 100644 index 0000000..b81d8eb Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-gzip.parquet new file mode 100644 index 0000000..e2ed52d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-lz4_raw.parquet new file mode 100644 index 0000000..5c2da8e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-snappy.parquet new file mode 100644 index 0000000..13c9fbb Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-uncompressed.parquet new file mode 100644 index 0000000..558a26b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-zstd.parquet new file mode 100644 index 0000000..da14832 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0058-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-brotli.parquet new file mode 100644 index 0000000..3b08fbe Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-gzip.parquet new file mode 100644 index 0000000..d56d734 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-lz4_raw.parquet new file mode 100644 index 0000000..d36e26a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-snappy.parquet new file mode 100644 index 0000000..da4a6a1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-uncompressed.parquet new file mode 100644 index 0000000..5dc9fe8 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-zstd.parquet new file mode 100644 index 0000000..75a500e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0062-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-brotli.parquet new file mode 100644 index 0000000..853da5c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-gzip.parquet new file mode 100644 index 0000000..55fbad2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-lz4_raw.parquet new file mode 100644 index 0000000..2702695 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-snappy.parquet new file mode 100644 index 0000000..2622c80 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-uncompressed.parquet new file mode 100644 index 0000000..6184d4d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-zstd.parquet new file mode 100644 index 0000000..c5abcc4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0096-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-brotli.parquet new file mode 100644 index 0000000..0b4f5ef Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-gzip.parquet new file mode 100644 index 0000000..12b6126 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-lz4_raw.parquet new file mode 100644 index 0000000..6ea5bb9 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-snappy.parquet new file mode 100644 index 0000000..79af3bd Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-uncompressed.parquet new file mode 100644 index 0000000..a3ca2b9 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-zstd.parquet new file mode 100644 index 0000000..07fa1fe Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0106-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-brotli.parquet new file mode 100644 index 0000000..65ba8c0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-gzip.parquet new file mode 100644 index 0000000..3d73337 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-lz4_raw.parquet new file mode 100644 index 0000000..f6243af Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-snappy.parquet new file mode 100644 index 0000000..93e352a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-uncompressed.parquet new file mode 100644 index 0000000..cc8527d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-zstd.parquet new file mode 100644 index 0000000..84e1cf4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0107-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-brotli.parquet new file mode 100644 index 0000000..86d0ac0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-gzip.parquet new file mode 100644 index 0000000..417d652 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-lz4_raw.parquet new file mode 100644 index 0000000..5950a89 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-snappy.parquet new file mode 100644 index 0000000..b04fd01 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-uncompressed.parquet new file mode 100644 index 0000000..db5a5ea Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-zstd.parquet new file mode 100644 index 0000000..62304bb Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0117-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-brotli.parquet new file mode 100644 index 0000000..0744008 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-gzip.parquet new file mode 100644 index 0000000..5f0f0a7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-lz4_raw.parquet new file mode 100644 index 0000000..dd70191 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-snappy.parquet new file mode 100644 index 0000000..4c4b141 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-uncompressed.parquet new file mode 100644 index 0000000..d6de52b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-zstd.parquet new file mode 100644 index 0000000..e27d5c3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0122-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-brotli.parquet new file mode 100644 index 0000000..f00b398 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-gzip.parquet new file mode 100644 index 0000000..7406720 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-lz4_raw.parquet new file mode 100644 index 0000000..31abced Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-snappy.parquet new file mode 100644 index 0000000..0d23d32 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-uncompressed.parquet new file mode 100644 index 0000000..8c7a8f5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-zstd.parquet new file mode 100644 index 0000000..bdd4069 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0123-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-brotli.parquet new file mode 100644 index 0000000..ec21f8a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-gzip.parquet new file mode 100644 index 0000000..7877c5e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-lz4_raw.parquet new file mode 100644 index 0000000..1d4f951 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-snappy.parquet new file mode 100644 index 0000000..39c8d43 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-uncompressed.parquet new file mode 100644 index 0000000..e614688 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-zstd.parquet new file mode 100644 index 0000000..eb2fc06 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0144-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-brotli.parquet new file mode 100644 index 0000000..04bc622 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-gzip.parquet new file mode 100644 index 0000000..890d167 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-lz4_raw.parquet new file mode 100644 index 0000000..769bd60 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-snappy.parquet new file mode 100644 index 0000000..115ef51 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-uncompressed.parquet new file mode 100644 index 0000000..c7669a5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-zstd.parquet new file mode 100644 index 0000000..a74bece Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0145-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-brotli.parquet new file mode 100644 index 0000000..92ac887 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-gzip.parquet new file mode 100644 index 0000000..dda994b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-lz4_raw.parquet new file mode 100644 index 0000000..67f3394 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-snappy.parquet new file mode 100644 index 0000000..dabbdb0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-uncompressed.parquet new file mode 100644 index 0000000..b2143a5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-zstd.parquet new file mode 100644 index 0000000..1802f93 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0148-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-brotli.parquet new file mode 100644 index 0000000..fa09929 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-gzip.parquet new file mode 100644 index 0000000..e34004a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-lz4_raw.parquet new file mode 100644 index 0000000..8a5ce9d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-snappy.parquet new file mode 100644 index 0000000..cb34285 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-uncompressed.parquet new file mode 100644 index 0000000..53b725c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-zstd.parquet new file mode 100644 index 0000000..bf33638 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0178-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-brotli.parquet new file mode 100644 index 0000000..6bae53d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-gzip.parquet new file mode 100644 index 0000000..fe7000f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-lz4_raw.parquet new file mode 100644 index 0000000..3b503a7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-snappy.parquet new file mode 100644 index 0000000..0f2203b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-uncompressed.parquet new file mode 100644 index 0000000..55da5c0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-zstd.parquet new file mode 100644 index 0000000..06f994f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0191-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-brotli.parquet new file mode 100644 index 0000000..9416543 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-gzip.parquet new file mode 100644 index 0000000..560ef71 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-lz4_raw.parquet new file mode 100644 index 0000000..2dfaa28 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-snappy.parquet new file mode 100644 index 0000000..991a211 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-uncompressed.parquet new file mode 100644 index 0000000..540d490 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-zstd.parquet new file mode 100644 index 0000000..2040998 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0202-v2-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-brotli.parquet new file mode 100644 index 0000000..384ad80 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-gzip.parquet new file mode 100644 index 0000000..7b4f3ba Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-lz4_raw.parquet new file mode 100644 index 0000000..0a53a8f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-snappy.parquet new file mode 100644 index 0000000..4a49c22 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-uncompressed.parquet new file mode 100644 index 0000000..c34b5a3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-zstd.parquet new file mode 100644 index 0000000..029d1bb Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0249-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-brotli.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-brotli.parquet new file mode 100644 index 0000000..0737c57 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-brotli.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-gzip.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-gzip.parquet new file mode 100644 index 0000000..c92f50c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-gzip.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-lz4_raw.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-lz4_raw.parquet new file mode 100644 index 0000000..71402d0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-lz4_raw.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-snappy.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-snappy.parquet new file mode 100644 index 0000000..f782d4f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-snappy.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-uncompressed.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-uncompressed.parquet new file mode 100644 index 0000000..391ac76 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-uncompressed.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-zstd.parquet b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-zstd.parquet new file mode 100644 index 0000000..ef1876c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/property/generated-0251-v1-zstd.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v1.parquet new file mode 100644 index 0000000..acde86a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v2.parquet new file mode 100644 index 0000000..566b09e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-duplicate-keys_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet new file mode 100644 index 0000000..059b8a5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet new file mode 100644 index 0000000..354ff88 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3_v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3_v1.parquet new file mode 100644 index 0000000..7e59a98 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3_v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3_v2.parquet new file mode 100644 index 0000000..7b584de Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-list-rule3_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v1.parquet new file mode 100644 index 0000000..07b1964 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v2.parquet new file mode 100644 index 0000000..30d8be7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/arrow-rs/arrow-rs-optional-key-present_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map.v1.parquet new file mode 100644 index 0000000..a3b2110 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map.v2.parquet new file mode 100644 index 0000000..a638506 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map_utf8.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map_utf8.v1.parquet new file mode 100644 index 0000000..05dab30 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map_utf8.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map_utf8.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map_utf8.v2.parquet new file mode 100644 index 0000000..356eac5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_direct_map_utf8.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule1_primitive.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule1_primitive.v1.parquet new file mode 100644 index 0000000..de1375e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule1_primitive.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule1_primitive.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule1_primitive.v2.parquet new file mode 100644 index 0000000..2d56460 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule1_primitive.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule2_struct.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule2_struct.v1.parquet new file mode 100644 index 0000000..7caa996 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule2_struct.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule2_struct.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule2_struct.v2.parquet new file mode 100644 index 0000000..21b8b7e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule2_struct.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_nested.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_nested.v1.parquet new file mode 100644 index 0000000..83813f1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_nested.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_nested.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_nested.v2.parquet new file mode 100644 index 0000000..c5de8e1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_nested.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet new file mode 100644 index 0000000..f768bf1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet new file mode 100644 index 0000000..9441eef Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_array.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_array.v1.parquet new file mode 100644 index 0000000..bb89595 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_array.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_array.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_array.v2.parquet new file mode 100644 index 0000000..41ebb78 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_array.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_tuple.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_tuple.v1.parquet new file mode 100644 index 0000000..9d95055 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_tuple.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_tuple.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_tuple.v2.parquet new file mode 100644 index 0000000..0447ba5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule4_tuple.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_extended.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_extended.v1.parquet new file mode 100644 index 0000000..4b697e7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_extended.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_extended.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_extended.v2.parquet new file mode 100644 index 0000000..d2aebbd Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_extended.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_paired.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_paired.v1.parquet new file mode 100644 index 0000000..4a0b004 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_paired.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_paired.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_paired.v2.parquet new file mode 100644 index 0000000..6caf12b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_optional_paired.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_required.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_required.v1.parquet new file mode 100644 index 0000000..a3eedb8 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_required.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_required.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_required.v2.parquet new file mode 100644 index 0000000..ecb0013 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/list_rule5_required.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_arbitrary_names.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_arbitrary_names.v1.parquet new file mode 100644 index 0000000..e04d6c1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_arbitrary_names.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_arbitrary_names.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_arbitrary_names.v2.parquet new file mode 100644 index 0000000..cc4b748 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_arbitrary_names.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_key_only.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_key_only.v1.parquet new file mode 100644 index 0000000..337949f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_key_only.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_key_only.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_key_only.v2.parquet new file mode 100644 index 0000000..20f9107 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_key_only.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standalone_mkv.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standalone_mkv.v1.parquet new file mode 100644 index 0000000..5205782 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standalone_mkv.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standalone_mkv.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standalone_mkv.v2.parquet new file mode 100644 index 0000000..193b07f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standalone_mkv.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standard.v1.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standard.v1.parquet new file mode 100644 index 0000000..bfb4bb2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standard.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standard.v2.parquet b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standard.v2.parquet new file mode 100644 index 0000000..4d2c160 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/julia/rewrite/parquet-java/map_standard.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-duplicate-keys_v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-duplicate-keys_v1.parquet new file mode 100644 index 0000000..7e55c51 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-duplicate-keys_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-duplicate-keys_v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-duplicate-keys_v2.parquet new file mode 100644 index 0000000..d51755e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-duplicate-keys_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet new file mode 100644 index 0000000..96470aa Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet new file mode 100644 index 0000000..5fb8dc2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3_v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3_v1.parquet new file mode 100644 index 0000000..3c23603 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3_v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3_v2.parquet new file mode 100644 index 0000000..78e4ea3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-list-rule3_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-optional-key-present_v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-optional-key-present_v1.parquet new file mode 100644 index 0000000..86ca36b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-optional-key-present_v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-optional-key-present_v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-optional-key-present_v2.parquet new file mode 100644 index 0000000..6e77cd4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/arrow-rs/arrow-rs-optional-key-present_v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map.v1.parquet new file mode 100644 index 0000000..89ad90b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map.v2.parquet new file mode 100644 index 0000000..aa948d6 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map_utf8.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map_utf8.v1.parquet new file mode 100644 index 0000000..6175972 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map_utf8.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map_utf8.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map_utf8.v2.parquet new file mode 100644 index 0000000..74b037f Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_direct_map_utf8.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule1_primitive.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule1_primitive.v1.parquet new file mode 100644 index 0000000..147de4e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule1_primitive.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule1_primitive.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule1_primitive.v2.parquet new file mode 100644 index 0000000..1892c07 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule1_primitive.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule2_struct.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule2_struct.v1.parquet new file mode 100644 index 0000000..0fd733b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule2_struct.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule2_struct.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule2_struct.v2.parquet new file mode 100644 index 0000000..660881e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule2_struct.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_nested.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_nested.v1.parquet new file mode 100644 index 0000000..d3e759c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_nested.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_nested.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_nested.v2.parquet new file mode 100644 index 0000000..f9d64ec Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_nested.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet new file mode 100644 index 0000000..b896c45 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet new file mode 100644 index 0000000..17e36fc Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_array.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_array.v1.parquet new file mode 100644 index 0000000..dae8409 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_array.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_array.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_array.v2.parquet new file mode 100644 index 0000000..9ea5d76 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_array.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_tuple.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_tuple.v1.parquet new file mode 100644 index 0000000..4f0b5aa Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_tuple.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_tuple.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_tuple.v2.parquet new file mode 100644 index 0000000..c7a5ea7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule4_tuple.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_extended.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_extended.v1.parquet new file mode 100644 index 0000000..b5c3dc2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_extended.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_extended.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_extended.v2.parquet new file mode 100644 index 0000000..07936fd Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_extended.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_paired.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_paired.v1.parquet new file mode 100644 index 0000000..5d0efa6 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_paired.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_paired.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_paired.v2.parquet new file mode 100644 index 0000000..69bff6e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_optional_paired.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_required.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_required.v1.parquet new file mode 100644 index 0000000..454666e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_required.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_required.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_required.v2.parquet new file mode 100644 index 0000000..ee2246c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/list_rule5_required.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_arbitrary_names.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_arbitrary_names.v1.parquet new file mode 100644 index 0000000..8d8065c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_arbitrary_names.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_arbitrary_names.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_arbitrary_names.v2.parquet new file mode 100644 index 0000000..c717ab0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_arbitrary_names.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_key_only.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_key_only.v1.parquet new file mode 100644 index 0000000..9d615fd Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_key_only.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_key_only.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_key_only.v2.parquet new file mode 100644 index 0000000..82f4240 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_key_only.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standalone_mkv.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standalone_mkv.v1.parquet new file mode 100644 index 0000000..5b76d0e Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standalone_mkv.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standalone_mkv.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standalone_mkv.v2.parquet new file mode 100644 index 0000000..0fb59a6 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standalone_mkv.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standard.v1.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standard.v1.parquet new file mode 100644 index 0000000..fb64921 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standard.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standard.v2.parquet b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standard.v2.parquet new file mode 100644 index 0000000..8d38ab3 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/external/parquet-java/map_standard.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/direct-list-of-map.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/direct-list-of-map.v1.parquet new file mode 100644 index 0000000..0f319d7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/direct-list-of-map.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/direct-list-of-map.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/direct-list-of-map.v2.parquet new file mode 100644 index 0000000..78c45b0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/direct-list-of-map.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-1.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-1.v1.parquet new file mode 100644 index 0000000..93ce741 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-1.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-1.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-1.v2.parquet new file mode 100644 index 0000000..9900034 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-1.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-2.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-2.v1.parquet new file mode 100644 index 0000000..a03b530 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-2.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-2.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-2.v2.parquet new file mode 100644 index 0000000..557cb49 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-2.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-3.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-3.v1.parquet new file mode 100644 index 0000000..4edf60c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-3.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-3.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-3.v2.parquet new file mode 100644 index 0000000..a3f8095 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-3.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-array.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-array.v1.parquet new file mode 100644 index 0000000..3e873cf Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-array.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-array.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-array.v2.parquet new file mode 100644 index 0000000..0c9f45a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-array.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-tuple.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-tuple.v1.parquet new file mode 100644 index 0000000..6ab4134 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-tuple.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-tuple.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-tuple.v2.parquet new file mode 100644 index 0000000..72c3ea7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-4-tuple.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-extended.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-extended.v1.parquet new file mode 100644 index 0000000..2aa005b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-extended.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-extended.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-extended.v2.parquet new file mode 100644 index 0000000..0378c02 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-extended.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-paired.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-paired.v1.parquet new file mode 100644 index 0000000..fcdc223 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-paired.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-paired.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-paired.v2.parquet new file mode 100644 index 0000000..2a8a2cb Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-paired.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-required.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-required.v1.parquet new file mode 100644 index 0000000..ebadd98 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-required.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-required.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-required.v2.parquet new file mode 100644 index 0000000..dc36776 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/list-rule-5-required.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/map-key-only.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/map-key-only.v1.parquet new file mode 100644 index 0000000..0dd42d5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/map-key-only.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/map-key-only.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/map-key-only.v2.parquet new file mode 100644 index 0000000..0697fee Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/map-key-only.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/map-optional-key.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/map-optional-key.v1.parquet new file mode 100644 index 0000000..324feb5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/map-optional-key.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/map-optional-key.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/map-optional-key.v2.parquet new file mode 100644 index 0000000..04a7f86 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/map-optional-key.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/map-standard.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/map-standard.v1.parquet new file mode 100644 index 0000000..3a32786 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/map-standard.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/map-standard.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/map-standard.v2.parquet new file mode 100644 index 0000000..47ef599 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/map-standard.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/schema-provenance.v1.parquet b/test/conformance/n5/julia-fixtures/reference/model/schema-provenance.v1.parquet new file mode 100644 index 0000000..a95f9e0 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/schema-provenance.v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/model/schema-provenance.v2.parquet b/test/conformance/n5/julia-fixtures/reference/model/schema-provenance.v2.parquet new file mode 100644 index 0000000..5d38b7c Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/model/schema-provenance.v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0002-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0002-v1.parquet new file mode 100644 index 0000000..91f03d5 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0002-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0004-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0004-v1.parquet new file mode 100644 index 0000000..2b3e6e2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0004-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0005-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0005-v2.parquet new file mode 100644 index 0000000..04dfe33 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0005-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0006-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0006-v1.parquet new file mode 100644 index 0000000..50a05b7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0006-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0007-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0007-v2.parquet new file mode 100644 index 0000000..893c05a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0007-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0008-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0008-v1.parquet new file mode 100644 index 0000000..6713730 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0008-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0009-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0009-v2.parquet new file mode 100644 index 0000000..03b2e3d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0009-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0010-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0010-v1.parquet new file mode 100644 index 0000000..68f5c89 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0010-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0011-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0011-v2.parquet new file mode 100644 index 0000000..5837d3b Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0011-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0014-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0014-v1.parquet new file mode 100644 index 0000000..c5f0794 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0014-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0015-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0015-v2.parquet new file mode 100644 index 0000000..2a2d03d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0015-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0018-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0018-v1.parquet new file mode 100644 index 0000000..a1393b7 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0018-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0028-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0028-v1.parquet new file mode 100644 index 0000000..b51b2c1 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0028-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0034-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0034-v2.parquet new file mode 100644 index 0000000..e3ae293 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0034-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0038-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0038-v2.parquet new file mode 100644 index 0000000..4e91833 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0038-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0048-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0048-v2.parquet new file mode 100644 index 0000000..e9c9ee6 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0048-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0058-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0058-v2.parquet new file mode 100644 index 0000000..98c6f05 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0058-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0062-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0062-v2.parquet new file mode 100644 index 0000000..e7413e2 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0062-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0096-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0096-v2.parquet new file mode 100644 index 0000000..bba64ae Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0096-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0106-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0106-v2.parquet new file mode 100644 index 0000000..929352a Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0106-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0107-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0107-v1.parquet new file mode 100644 index 0000000..d331ed8 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0107-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0117-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0117-v1.parquet new file mode 100644 index 0000000..6215eaa Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0117-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0122-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0122-v2.parquet new file mode 100644 index 0000000..cef8049 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0122-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0123-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0123-v1.parquet new file mode 100644 index 0000000..1ba1f05 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0123-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0144-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0144-v2.parquet new file mode 100644 index 0000000..929cfac Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0144-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0145-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0145-v1.parquet new file mode 100644 index 0000000..688112d Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0145-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0148-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0148-v2.parquet new file mode 100644 index 0000000..4953282 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0148-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0178-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0178-v1.parquet new file mode 100644 index 0000000..36cb7ca Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0178-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0191-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0191-v2.parquet new file mode 100644 index 0000000..c861dfc Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0191-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0202-v2.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0202-v2.parquet new file mode 100644 index 0000000..46f02f4 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0202-v2.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0249-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0249-v1.parquet new file mode 100644 index 0000000..c40fe18 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0249-v1.parquet differ diff --git a/test/conformance/n5/julia-fixtures/reference/property/generated-0251-v1.parquet b/test/conformance/n5/julia-fixtures/reference/property/generated-0251-v1.parquet new file mode 100644 index 0000000..a74e632 Binary files /dev/null and b/test/conformance/n5/julia-fixtures/reference/property/generated-0251-v1.parquet differ diff --git a/test/conformance/n5/manifest.toml b/test/conformance/n5/manifest.toml new file mode 100644 index 0000000..b302000 --- /dev/null +++ b/test/conformance/n5/manifest.toml @@ -0,0 +1,294 @@ +manifest_version = 1 + +[[expected_evidence]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +scope = "owned" +file = "expected/parquet-java.jsonl" +sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +fixture_count = 30 + +[[expected_evidence]] +producer = "arrow-rs" +producer_version = "59.2.0" +producer_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +scope = "owned" +file = "expected/arrow-rs.json" +sha256 = "12f821dac1a2f2187302947c505d2656b9da05d936c5fcdc08ac11e1deff7fba" +fixture_count = 6 + +[[expected_evidence]] +producer = "arrow-rs" +producer_version = "59.2.0" +producer_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +scope = "near-neighbor" +file = "expected/arrow-rs-rule3-near-neighbor.json" +sha256 = "e1469a653fc5e8f26db3d111d4af366c90aff24403912c7bc20b8a47b16dd16e" +fixture_count = 2 + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_direct_map" +role = "compatibility" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_direct_map.v1.parquet", sha256 = "c84b4c582a00245acbc6330c80a3437f0007e1c2311adf31025b6952a95999e3" }, + { page_version = "v2", file = "golden/parquet-java/list_direct_map.v2.parquet", sha256 = "c2d8ac178692160961f93b2a572a0bd4b19f71c63eddb56f7585c3e5a30c13eb" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_direct_map_utf8" +role = "compatibility" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_direct_map_utf8.v1.parquet", sha256 = "4f7faf3e2bef6c043d15e88feeb796728df51a9648abb81a301e0f6e6f97fd0b" }, + { page_version = "v2", file = "golden/parquet-java/list_direct_map_utf8.v2.parquet", sha256 = "edc66d338474404c9c7db8a4b93d5f1def987b4883f5aefe7d042c348aa282de" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule1_primitive" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule1_primitive.v1.parquet", sha256 = "3466fa00347fae5832a8b751cd50d74e0af0df490d7262bdccf36f0c2f38c2db" }, + { page_version = "v2", file = "golden/parquet-java/list_rule1_primitive.v2.parquet", sha256 = "056989f7d112e735fc4f4e835278ba58610e522e1aa3a3e05afdaf540c0463cd" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule2_struct" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule2_struct.v1.parquet", sha256 = "b3c6ddd69b4b75bfe0ba1fcb21aca6db5d7b7c083fda0cd106b7229baa0a2f7b" }, + { page_version = "v2", file = "golden/parquet-java/list_rule2_struct.v2.parquet", sha256 = "2b059b3b5b375c736e7d0abb69a5f1e8dc48606008ffd9e3910f34d3543fc6a6" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule3_nested" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule3_nested.v1.parquet", sha256 = "c85821a7f6593ad3092f2efd3c8acc602f4969eb8dc36d49f1868b22f1d7d56a" }, + { page_version = "v2", file = "golden/parquet-java/list_rule3_nested.v2.parquet", sha256 = "5333c37c239ae57781c63eec972537eb428e275da8d3cca0634acb963989e73e" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule3_unannotated_diagnostic" +role = "diagnostic" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule3_unannotated_diagnostic.v1.parquet", sha256 = "ab2c1a962ee992845540c6aa04056f15cdc4b40cf89627fb5160dd3fb07a5123" }, + { page_version = "v2", file = "golden/parquet-java/list_rule3_unannotated_diagnostic.v2.parquet", sha256 = "51ece6eccb1123e817c7e4a9f5f0ff235299501c7e7aa4e78f2878ecb4ae4e5f" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule4_array" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule4_array.v1.parquet", sha256 = "4b7a1f524f1f04a6fcb2f8ebabf1f82cf33d709cc154207e43e95ab61d09ed1a" }, + { page_version = "v2", file = "golden/parquet-java/list_rule4_array.v2.parquet", sha256 = "966d39ae1c7d1c011b3e2405ef0be9ff9d23372516abcc2060331b61c00bc642" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule4_tuple" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule4_tuple.v1.parquet", sha256 = "2b7e4ae67a92182f44cff1c584096bc6b6d741106fe56eb20b0e2d3a41d0a170" }, + { page_version = "v2", file = "golden/parquet-java/list_rule4_tuple.v2.parquet", sha256 = "9d51d23da84416b0b520e0fdb80796d80923b5c8cbdeeea454a8c422a897934a" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule5_optional_extended" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule5_optional_extended.v1.parquet", sha256 = "5281d05e2317fbc30e280c18b15078d0676b5ef22b1ee56d9788dc2e45d45af0" }, + { page_version = "v2", file = "golden/parquet-java/list_rule5_optional_extended.v2.parquet", sha256 = "e2b4bca7e67bac4dafdca1a4aea01d3bdf764a6ece37419edd2c39699efec6c4" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule5_optional_paired" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule5_optional_paired.v1.parquet", sha256 = "2873a69f47bd82a93625a78fda96a58a5783c5f473f0ace861216df876c9a725" }, + { page_version = "v2", file = "golden/parquet-java/list_rule5_optional_paired.v2.parquet", sha256 = "a465d3f92575429b545d4edad4aece7dd30019860ee71af75f61d035c0b00f04" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "list_rule5_required" +role = "binding" +rows = 4 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/list_rule5_required.v1.parquet", sha256 = "e16a41a44e974c44f8ca83021ecd55e016018bab7f86a930b2560af7d66de762" }, + { page_version = "v2", file = "golden/parquet-java/list_rule5_required.v2.parquet", sha256 = "a730dbdfca94503ce0f1e6717f96c4636378b2b7f56d593bebae86fa86513127" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "map_arbitrary_names" +role = "binding" +rows = 5 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/map_arbitrary_names.v1.parquet", sha256 = "0a6aadfd449407c0e6036d6b894e4f73c9d6a9b7a64b35eba16da2a685954ccb" }, + { page_version = "v2", file = "golden/parquet-java/map_arbitrary_names.v2.parquet", sha256 = "b69c7d382417e2be0834bbbc060a5a65548867c659aec51283b77650bccd8de4" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "map_key_only" +role = "compatibility" +rows = 3 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/map_key_only.v1.parquet", sha256 = "e05e566da94fcdd96d5ca17c4f8f4403541a50c94002bef15639df889fb79102" }, + { page_version = "v2", file = "golden/parquet-java/map_key_only.v2.parquet", sha256 = "3b6d03072c1046e000097bad8392319678256610a1ad56e7f6611c390cb0a5ef" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "map_standalone_mkv" +role = "compatibility" +rows = 5 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/map_standalone_mkv.v1.parquet", sha256 = "a99a78cc84bdc17a30d0ab73c28c5edf6a7640b85ceab9470fedaa7369a5f401" }, + { page_version = "v2", file = "golden/parquet-java/map_standalone_mkv.v2.parquet", sha256 = "467cdc8abf6514d94aa8698d7b9bcad654684bd5d465cc0b973329a3c9db1f30" }, +] + +[[case]] +producer = "parquet-java" +producer_version = "1.17.1" +producer_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +case_id = "map_standard" +role = "binding" +rows = 5 +expected_evidence_file = "expected/parquet-java.jsonl" +expected_evidence_sha256 = "2b55fa484426c576949095da45e48ba24a34499ad55b28d05e55e34987294671" +files = [ + { page_version = "v1", file = "golden/parquet-java/map_standard.v1.parquet", sha256 = "0acccffff97f7ff6867daaa35f57ad1a9609ed3407e81921a275f9c3f40d20d5" }, + { page_version = "v2", file = "golden/parquet-java/map_standard.v2.parquet", sha256 = "136bf8f4949fa97f52b337b7a4c349d6fb2ca2f38e2df815adadea5df134c9fd" }, +] + +[[case]] +producer = "arrow-rs" +producer_version = "59.2.0" +producer_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +case_id = "arrow-rs-duplicate-keys" +role = "binding" +rows = 5 +expected_evidence_file = "expected/arrow-rs.json" +expected_evidence_sha256 = "12f821dac1a2f2187302947c505d2656b9da05d936c5fcdc08ac11e1deff7fba" +files = [ + { page_version = "v1", file = "golden/arrow-rs/arrow-rs-duplicate-keys_v1.parquet", sha256 = "acf5536a16167a2a870dfd55ad2a5d208712069d8e0f034cf396fa6d6c193184" }, + { page_version = "v2", file = "golden/arrow-rs/arrow-rs-duplicate-keys_v2.parquet", sha256 = "baaf9a09bc397db8c0b1d0453fa2d87e36f6a74bbc56778be39d27304007cbb3" }, +] + +[[case]] +producer = "arrow-rs" +producer_version = "59.2.0" +producer_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +case_id = "arrow-rs-list-rule3" +role = "binding" +rows = 4 +expected_evidence_file = "expected/arrow-rs.json" +expected_evidence_sha256 = "12f821dac1a2f2187302947c505d2656b9da05d936c5fcdc08ac11e1deff7fba" +files = [ + { page_version = "v1", file = "golden/arrow-rs/arrow-rs-list-rule3_v1.parquet", sha256 = "916475c20dde6afe338d60e43d2e4a6293c2bc26efb6dd439d1d7d9299834bc6" }, + { page_version = "v2", file = "golden/arrow-rs/arrow-rs-list-rule3_v2.parquet", sha256 = "ef72761c509f8bc302986d130f6d7d9382b975c8a3462a955c599bd5f427edac" }, +] + +[[case]] +producer = "arrow-rs" +producer_version = "59.2.0" +producer_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +case_id = "arrow-rs-optional-key-present" +role = "compatibility" +rows = 4 +expected_evidence_file = "expected/arrow-rs.json" +expected_evidence_sha256 = "12f821dac1a2f2187302947c505d2656b9da05d936c5fcdc08ac11e1deff7fba" +files = [ + { page_version = "v1", file = "golden/arrow-rs/arrow-rs-optional-key-present_v1.parquet", sha256 = "a33be66fe1a5e7189c457b49163e5a4b658228ffb5054a40cd22f20f731ba21a" }, + { page_version = "v2", file = "golden/arrow-rs/arrow-rs-optional-key-present_v2.parquet", sha256 = "f010b919bcc2d4e8c42f6d97df651266de06b6352f6b993756fd0a007e1c5028" }, +] + +[[case]] +producer = "arrow-rs" +producer_version = "59.2.0" +producer_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +case_id = "arrow-rs-list-rule3-unannotated-near-neighbor" +role = "diagnostic" +rows = 4 +expected_evidence_file = "expected/arrow-rs-rule3-near-neighbor.json" +expected_evidence_sha256 = "e1469a653fc5e8f26db3d111d4af366c90aff24403912c7bc20b8a47b16dd16e" +files = [ + { page_version = "v1", file = "golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v1.parquet", sha256 = "40eb4521da12a9bd5db38cfe9e1f31dd91bd1095ec5b24b94f9a02318e40128f" }, + { page_version = "v2", file = "golden/arrow-rs/arrow-rs-list-rule3-unannotated-near-neighbor_v2.parquet", sha256 = "4fe0319ea46248612489d845eb6fdbfa127077f06a9e7a2efc14fa67def66ce1" }, +] diff --git a/test/conformance/n5/model/N5ConformanceModel.jl b/test/conformance/n5/model/N5ConformanceModel.jl new file mode 100644 index 0000000..b1842ad --- /dev/null +++ b/test/conformance/n5/model/N5ConformanceModel.jl @@ -0,0 +1,16 @@ +module N5ConformanceModel + +using Parquet +using SHA + +const MD = Parquet.Metadata +const TH = Parquet.Thrift + +include("model.jl") +include("wire.jl") +include("goldens.jl") +include("properties.jl") +include("manifest.jl") +include("integration.jl") + +end diff --git a/test/conformance/n5/model/goldens.jl b/test/conformance/n5/model/goldens.jl new file mode 100644 index 0000000..35b1818 --- /dev/null +++ b/test/conformance/n5/model/goldens.jl @@ -0,0 +1,705 @@ +struct N5Golden + name::String + node::N5Node + rows::Vector{Any} + schema::Vector{MD.SchemaElement} + paths::Vector{Vector{String}} + streams::Vector{N5LeafStream} +end + +struct N5GeneratedCase + name::String + node::N5Node + rows::Vector{Any} +end + +struct N5SchemaControl + name::String + schema::Vector{MD.SchemaElement} + streams::Vector{N5LeafStream} + rows::Vector{Any} + expected::Symbol + node::Union{Nothing,N5Node} +end + +struct N5SchemaFailure + name::String + schema::Vector{MD.SchemaElement} + streams::Vector{N5LeafStream} +end + +function n5stream(repetition, definition, values, maxrepetition::Integer, + maxdefinition::Integer) + return N5LeafStream(UInt64[repetition...], UInt64[definition...], + Any[values...], Int(maxrepetition), Int(maxdefinition)) +end + +function _n5golden(name::AbstractString, node::N5Node, rows, + paths::Vector{Vector{String}}, streams::Vector{N5LeafStream}) + return N5Golden(String(name), node, Any[rows...], n5schema(node), paths, + streams) +end + +function _n5rule1() + element = n5primitive("element", MD.Type.INT32) + node = n5list("items", element; optional=true, layout=:rule1, + annotation=:dual) + rows = Any[missing, [], Any[Int32(10)], Any[Int32(20), Int32(30)]] + streams = N5LeafStream[n5stream([0, 0, 0, 0, 1], + [0, 1, 2, 2, 2], Int32[10, 20, 30], 1, 2)] + return _n5golden("list-rule-1", node, rows, + [String["items", "element"]], streams) +end + +function _n5rule2() + fields = N5Node[ + n5primitive("x", MD.Type.INT32), + n5primitive("y", MD.Type.INT32; optional=true), + ] + element = n5struct("element", fields) + node = n5list("items", element; optional=true, layout=:rule2, + wrapper="element", annotation=:dual) + rows = Any[ + missing, + [], + Any[n5record(Int32(1), missing)], + Any[n5record(Int32(2), Int32(20)), + n5record(Int32(3), Int32(30))], + ] + streams = N5LeafStream[ + n5stream([0, 0, 0, 0, 1], [0, 1, 2, 2, 2], + Int32[1, 2, 3], 1, 2), + n5stream([0, 0, 0, 0, 1], [0, 1, 2, 3, 3], + Int32[20, 30], 1, 3), + ] + paths = [String["items", "element", "x"], + String["items", "element", "y"]] + return _n5golden("list-rule-2", node, rows, paths, streams) +end + +function _n5rule3() + inner = n5list("array", n5primitive("array", MD.Type.INT32); + layout=:rule1, annotation=:dual) + node = n5list("items", inner; optional=true, layout=:rule3, + wrapper="array", annotation=:dual) + rows = Any[ + missing, + [], + Any[[]], + Any[Any[Int32(1), Int32(2)], [], Any[Int32(3)]], + ] + streams = N5LeafStream[n5stream([0, 0, 0, 0, 2, 1, 1], + [0, 1, 2, 3, 3, 2, 3], Int32[1, 2, 3], 2, 3)] + return _n5golden("list-rule-3", node, rows, + [String["items", "array", "array"]], streams) +end + +function _n5rule4array() + element = n5struct("array", N5Node[ + n5primitive("v", MD.Type.INT32; optional=true), + ]) + node = n5list("items", element; optional=true, layout=:rule4, + wrapper="array", annotation=:dual) + rows = Any[ + missing, + [], + Any[n5record(missing)], + Any[n5record(Int32(4)), n5record(missing)], + ] + streams = N5LeafStream[n5stream([0, 0, 0, 0, 1], + [0, 1, 2, 3, 2], Int32[4], 1, 3)] + return _n5golden("list-rule-4-array", node, rows, + [String["items", "array", "v"]], streams) +end + +function _n5rule4tuple() + element = n5struct("items_tuple", N5Node[ + n5primitive("v", MD.Type.INT32; optional=true), + ]) + node = n5list("items", element; optional=true, layout=:rule4, + wrapper="items_tuple", annotation=:dual) + rows = Any[ + missing, + [], + Any[n5record(Int32(7))], + Any[n5record(missing), n5record(Int32(8))], + ] + streams = N5LeafStream[n5stream([0, 0, 0, 0, 1], + [0, 1, 3, 2, 3], Int32[7, 8], 1, 3)] + return _n5golden("list-rule-4-tuple", node, rows, + [String["items", "items_tuple", "v"]], streams) +end + +function _n5rule5required() + element = n5primitive("value", MD.Type.INT32) + node = n5list("items", element; optional=true, layout=:rule5, + wrapper="element", annotation=:dual) + rows = Any[missing, [], Any[Int32(10)], Any[Int32(20), Int32(30)]] + streams = N5LeafStream[n5stream([0, 0, 0, 0, 1], + [0, 1, 2, 2, 2], Int32[10, 20, 30], 1, 2)] + return _n5golden("list-rule-5-required", node, rows, + [String["items", "element", "value"]], streams) +end + +function _n5rule5paired() + element = n5primitive("value", MD.Type.INT32; optional=true) + node = n5list("items", element; optional=true, layout=:rule5, + wrapper="element", annotation=:dual) + rows = Any[ + missing, + [], + Any[missing], + Any[Int32(4), missing], + ] + streams = N5LeafStream[n5stream([0, 0, 0, 0, 1], + [0, 1, 2, 3, 2], Int32[4], 1, 3)] + return _n5golden("list-rule-5-paired", node, rows, + [String["items", "element", "value"]], streams) +end + +function _n5rule5extended() + element = n5primitive("value", MD.Type.INT32; optional=true) + node = n5list("items", element; optional=true, layout=:rule5, + wrapper="element", annotation=:dual) + rows = Any[ + missing, + [], + Any[missing], + Any[Int32(5), missing, Int32(6)], + ] + streams = N5LeafStream[n5stream([0, 0, 0, 0, 1, 1], + [0, 1, 2, 3, 2, 3], Int32[5, 6], 1, 3)] + return _n5golden("list-rule-5-extended", node, rows, + [String["items", "element", "value"]], streams) +end + +function _n5listofmap() + mapnode = n5map("map", n5primitive("key", MD.Type.INT32), + n5primitive("value", MD.Type.INT32); marker=:legacy, + entrymarker=:marked, entryname="entries") + node = n5list("items", mapnode; optional=true, layout=:rule3, + wrapper="map", annotation=:legacy) + rows = Any[ + missing, + [], + Any[n5mapvalue()], + Any[ + n5mapvalue(Int32(1) => Int32(10), Int32(1) => Int32(20)), + n5mapvalue(), + n5mapvalue(Int32(2) => Int32(30)), + ], + ] + streams = N5LeafStream[ + n5stream([0, 0, 0, 0, 2, 1, 1], [0, 1, 2, 3, 3, 2, 3], + Int32[1, 1, 2], 2, 3), + n5stream([0, 0, 0, 0, 2, 1, 1], [0, 1, 2, 3, 3, 2, 3], + Int32[10, 20, 30], 2, 3), + ] + paths = [String["items", "map", "entries", "key"], + String["items", "map", "entries", "value"]] + return _n5golden("direct-list-of-map", node, rows, paths, streams) +end + +function _n5standardmap() + key = n5primitive("key", MD.Type.BYTE_ARRAY; logical=:string) + value = n5primitive("value", MD.Type.INT32; optional=true) + node = n5map("attrs", key, value; optional=true, marker=:dual, + entrymarker=:marked) + rows = Any[ + missing, + n5mapvalue(), + n5mapvalue("a" => missing), + n5mapvalue("a" => Int32(1), "a" => Int32(2), + "b" => Int32(3)), + n5mapvalue("c" => Int32(4)), + ] + streams = N5LeafStream[ + n5stream([0, 0, 0, 0, 1, 1, 0], [0, 1, 2, 2, 2, 2, 2], + ["a", "a", "a", "b", "c"], 1, 2), + n5stream([0, 0, 0, 0, 1, 1, 0], [0, 1, 2, 3, 3, 3, 3], + Int32[1, 2, 3, 4], 1, 3), + ] + paths = [String["attrs", "key_value", "key"], + String["attrs", "key_value", "value"]] + return _n5golden("map-standard", node, rows, paths, streams) +end + +function _n5keyonlymap() + key = n5primitive("key", MD.Type.BYTE_ARRAY; logical=:string) + node = n5map("attrs", key, nothing; marker=:dual, + entrymarker=:marked) + rows = Any[ + n5mapvalue(), + n5keyset("k1"), + n5keyset("k2", "k2"), + ] + streams = N5LeafStream[n5stream([0, 0, 0, 1], [0, 1, 1, 1], + ["k1", "k2", "k2"], 1, 1)] + return _n5golden("map-key-only", node, rows, + [String["attrs", "key_value", "key"]], streams) +end + +function _n5optionalkeymap() + key = n5primitive("key", MD.Type.BYTE_ARRAY; optional=true, + logical=:string) + value = n5primitive("value", MD.Type.INT32) + node = n5map("attrs", key, value; optional=true, marker=:legacy, + entrymarker=:marked) + rows = Any[ + missing, + n5mapvalue(), + n5mapvalue("a" => Int32(1)), + n5mapvalue("b" => Int32(2), "c" => Int32(3)), + ] + streams = N5LeafStream[ + n5stream([0, 0, 0, 0, 1], [0, 1, 3, 3, 3], + ["a", "b", "c"], 1, 3), + n5stream([0, 0, 0, 0, 1], [0, 1, 2, 2, 2], + Int32[1, 2, 3], 1, 2), + ] + paths = [String["attrs", "key_value", "key"], + String["attrs", "key_value", "value"]] + return _n5golden("map-optional-key", node, rows, paths, streams) +end + +function n5bindinggoldens() + return N5Golden[ + _n5rule1(), + _n5rule2(), + _n5rule3(), + _n5rule4array(), + _n5rule4tuple(), + _n5rule5required(), + _n5rule5paired(), + _n5rule5extended(), + _n5listofmap(), + _n5standardmap(), + _n5keyonlymap(), + _n5optionalkeymap(), + ] +end + +function n5recursivecases() + structkey = n5struct("key", N5Node[ + n5primitive("id", MD.Type.INT32), + n5primitive("tag", MD.Type.INT32; optional=true), + ]) + listvalue = n5list("value", n5primitive("element", MD.Type.INT32); + optional=true) + structlist = n5map("attrs", structkey, listvalue; optional=true, + marker=:dual, entrymarker=:marked) + structlistrows = Any[ + missing, + n5mapvalue(), + n5mapvalue(n5record(Int32(1), missing) => + Any[Int32(10), Int32(20)], + n5record(Int32(2), Int32(7)) => missing), + ] + + listkey = n5list("key", n5primitive("element", MD.Type.INT32)) + nestedmap = n5map("value", + n5primitive("nested_key", MD.Type.BYTE_ARRAY; logical=:string), + n5primitive("nested_value", MD.Type.INT32; optional=true); + optional=true, marker=:dual, entrymarker=:marked) + listmap = n5map("attrs", listkey, nestedmap; optional=true, + marker=:modern_alias, entrymarker=:future_marked) + listmaprows = Any[ + missing, + n5mapvalue(), + n5mapvalue(Any[Int32(1), Int32(2)] => + n5mapvalue("a" => Int32(1), "a" => Int32(2)), + Any[] => missing), + ] + + mapkey = n5map("key", n5primitive("inner_key", MD.Type.INT32), + n5primitive("inner_value", MD.Type.INT32; optional=true); + marker=:legacy, entrymarker=:marked) + structvalue = n5struct("value", N5Node[ + n5primitive("payload", MD.Type.INT32), + n5list("items", n5primitive("element", MD.Type.INT32); + optional=true), + ]; optional=true) + mapstruct = n5map("attrs", mapkey, structvalue; optional=true, + marker=:modern_primitive, entrymarker=:empty) + mapstructrows = Any[ + missing, + n5mapvalue(), + n5mapvalue(n5mapvalue(Int32(1) => Int32(10), + Int32(1) => missing) => + n5record(Int32(5), Any[Int32(6), Int32(7)]), + n5mapvalue() => missing), + ] + return N5GeneratedCase[ + N5GeneratedCase("recursive-struct-key-list-value", structlist, + structlistrows), + N5GeneratedCase("recursive-list-key-map-value", listmap, + listmaprows), + N5GeneratedCase("recursive-map-key-struct-value", mapstruct, + mapstructrows), + ] +end + +function _n5matrixrows(optional::Bool, valuemode::Symbol) + empty = n5mapvalue() + present = if valuemode === :absent + n5keyset("a", "a", "b") + elseif valuemode === :required + n5mapvalue("a" => Int32(1), "a" => Int32(2), "b" => Int32(3)) + else + n5mapvalue("a" => missing, "a" => Int32(2), "b" => Int32(3)) + end + return optional ? Any[missing, empty, present] : Any[empty, present] +end + +function n5mapmatrix() + cases = N5GeneratedCase[] + for marker in (:modern, :dual, :modern_alias, :conflict, + :modern_primitive, :modern_unknown, :legacy, :alias) + for optional in (false, true) + for entrymarker in (:none, :marked, :empty, :future, + :future_marked, :unknown_converted) + for optionalkey in (false, true) + for valuemode in (:absent, :required, :optional) + for arbitrary in (false, true) + entryname = arbitrary ? "entries_any" : "key_value" + keyname = arbitrary ? "left" : "key" + valuename = arbitrary ? "right" : "value" + key = n5primitive(keyname, MD.Type.BYTE_ARRAY; + optional=optionalkey, logical=:string) + value = valuemode === :absent ? nothing : + n5primitive(valuename, MD.Type.INT32; + optional=valuemode === :optional) + node = n5map("attrs", key, value; + optional=optional, marker=marker, + entrymarker=entrymarker, + entryname=entryname) + name = join((marker, optional ? :optional : :required, + entrymarker, optionalkey ? :optional_key : :required_key, + valuemode, arbitrary ? :arbitrary : :canonical), "-") + push!(cases, N5GeneratedCase(name, node, + _n5matrixrows(optional, valuemode))) + end + end + end + end + end + end + return cases +end + +function n5annotationcontrols() + cases = N5GeneratedCase[] + rows = Any[missing, [], Any[Int32(1), Int32(2)]] + for annotation in (:modern, :legacy, :dual, :conflict) + node = n5list("items", n5primitive("element", MD.Type.INT32); + optional=true, layout=:rule1, annotation=annotation) + push!(cases, N5GeneratedCase("list-annotation-$annotation", node, rows)) + end + maprows = Any[missing, n5mapvalue(), n5mapvalue("a" => Int32(1))] + for marker in (:modern, :legacy, :alias, :dual, :conflict, + :modern_alias, :modern_primitive, :modern_unknown) + node = n5map("attrs", + n5primitive("key", MD.Type.BYTE_ARRAY; logical=:string), + n5primitive("value", MD.Type.INT32); optional=true, + marker=marker, entrymarker=:marked) + push!(cases, N5GeneratedCase("map-annotation-$marker", node, maprows)) + end + return cases +end + +function _n5element(element::MD.SchemaElement; + type_=element.type_, type_length=element.type_length, + repetition_type=element.repetition_type, name=element.name, + num_children=element.num_children, + converted_type=element.converted_type, scale=element.scale, + precision=element.precision, field_id=element.field_id, + logicalType=element.logicalType, + unknown_fields=element.unknown_fields) + return MD.SchemaElement(type_=type_, type_length=type_length, + repetition_type=repetition_type, name=name, + num_children=num_children, converted_type=converted_type, + scale=scale, precision=precision, field_id=field_id, + logicalType=logicalType, unknown_fields=unknown_fields) +end + +function _n5variantlogical() + return MD.LogicalType(VARIANT=MD.VariantType(specification_version=Int8(1))) +end + +function _n5rawi32(id::Integer, value::Integer) + writer = TH.Writer() + TH.writei32!(writer, Int32(value)) + return TH.RawField(id, TH.I32, writer.buffer) +end + +function n5provenancegolden() + base = _n5rule1() + schema = copy(base.schema) + root = schema[1] + schema[1] = _n5element(root; field_id=Int32(101), + unknown_fields=(root.unknown_fields..., _n5rawi32(90, 900))) + outer = schema[2] + listtype = MD.ListType(unknown_fields=(_n5rawi32(91, 910),)) + schema[2] = _n5element(outer; field_id=Int32(102), + logicalType=MD.LogicalType(LIST=listtype), + unknown_fields=(outer.unknown_fields..., _n5rawi32(92, 920))) + leaf = schema[3] + schema[3] = _n5element(leaf; field_id=Int32(103), + logicalType=_n5futurelogical(Int16(2556)), + unknown_fields=(leaf.unknown_fields..., _n5rawi32(93, 930))) + return N5Golden("schema-provenance", base.node, base.rows, schema, + base.paths, base.streams) +end + +function _n5entrycontrol(name::String, logical, converted) + base = _n5standardmap() + schema = copy(base.schema) + schema[3] = _n5element(schema[3]; logicalType=logical, + converted_type=converted) + return N5SchemaControl(name, schema, base.streams, base.rows, :map, + base.node) +end + +function _n5mapentrynode(node::N5Map) + fields = N5Node[node.key] + node.value === nothing || push!(fields, node.value) + return n5struct(node.entryname, fields) +end + +function _n5mapentryrows(node::N5Map, rows::Vector{Any}) + output = Any[] + sizehint!(output, length(rows)) + for row in rows + if ismissing(row) + push!(output, missing) + continue + end + entries = Any[] + sizehint!(entries, length(row.entries)) + for entry in row.entries + values = Any[entry.key] + node.value === nothing || push!(values, entry.value) + push!(entries, N5Record(values)) + end + push!(output, entries) + end + return output +end + +function _n5mapalternate(node::N5Map, rows::Vector{Any}, expected::Symbol) + entries = _n5mapentrynode(node) + entryrows = _n5mapentryrows(node, rows) + if expected === :list + semantic = n5list(node.name, entries; optional=node.optional, + layout=:rule2, wrapper=node.entryname) + return semantic, entryrows + elseif expected === :struct + repeated = n5list(node.entryname, entries; layout=:rule2, + wrapper=node.entryname) + semantic = n5struct(node.name, N5Node[repeated]; optional=node.optional) + structrows = Any[ismissing(row) ? missing : n5record(row) + for row in entryrows] + return semantic, structrows + end + throw(ArgumentError("unsupported N5 alternate MAP meaning $expected")) +end + +function _n5outercontrol(name::String, logical, converted, + expected::Symbol; node::Union{Nothing,N5Node}=nothing) + base = _n5standardmap() + schema = copy(base.schema) + schema[2] = _n5element(schema[2]; logicalType=logical, + converted_type=converted) + expected === :map || (schema[3] = _n5element(schema[3]; + logicalType=nothing, converted_type=nothing)) + if node !== nothing + semantic = node + rows = base.rows + elseif expected === :map + semantic = base.node + rows = base.rows + else + semantic, rows = _n5mapalternate(base.node, base.rows, expected) + end + return N5SchemaControl(name, schema, base.streams, rows, expected, semantic) +end + +function n5mapbindingcontrols() + maplogical = MD.LogicalType(MAP=MD.MapType()) + listlogical = MD.LogicalType(LIST=MD.ListType()) + controls = N5SchemaControl[ + _n5outercontrol("outer-modern-map", maplogical, nothing, :map), + _n5outercontrol("outer-modern-map-matching", maplogical, + MD.ConvertedType.MAP, :map), + _n5outercontrol("outer-modern-map-alias", maplogical, + MD.ConvertedType.MAP_KEY_VALUE, :map), + _n5outercontrol("outer-modern-map-conflict", maplogical, + MD.ConvertedType.LIST, :map), + _n5outercontrol("outer-modern-map-primitive", maplogical, + MD.ConvertedType.UTF8, :map), + _n5outercontrol("outer-modern-map-unknown-converted", maplogical, + MD.ConvertedType.T(99), :map), + _n5outercontrol("outer-legacy-map", nothing, + MD.ConvertedType.MAP, :map), + _n5outercontrol("outer-legacy-map-alias", nothing, + MD.ConvertedType.MAP_KEY_VALUE, :map), + _n5outercontrol("outer-unknown-blocks-map", _n5futurelogical(), + MD.ConvertedType.MAP, :struct), + _n5outercontrol("outer-unknown-blocks-alias", _n5futurelogical(), + MD.ConvertedType.MAP_KEY_VALUE, :struct), + _n5outercontrol("outer-unannotated", nothing, nothing, :struct), + _n5outercontrol("outer-unknown-converted", nothing, + MD.ConvertedType.T(99), :struct), + _n5outercontrol("outer-list-wins-map", listlogical, + MD.ConvertedType.MAP, :list), + _n5outercontrol("outer-list-wins-alias", listlogical, + MD.ConvertedType.MAP_KEY_VALUE, :list), + _n5outercontrol("outer-variant-wins", _n5variantlogical(), + MD.ConvertedType.MAP, :struct), + _n5outercontrol("outer-empty-wins", MD.LogicalType(), + MD.ConvertedType.MAP, :struct), + _n5outercontrol("outer-converted-list", nothing, + MD.ConvertedType.LIST, :list), + _n5entrycontrol("entry-unmarked", nothing, nothing), + _n5entrycontrol("entry-map-key-value", nothing, + MD.ConvertedType.MAP_KEY_VALUE), + _n5entrycontrol("entry-empty-logical", MD.LogicalType(), nothing), + _n5entrycontrol("entry-unknown-logical", _n5futurelogical(), nothing), + _n5entrycontrol("entry-unknown-blocks-map-key-value", + _n5futurelogical(), MD.ConvertedType.MAP_KEY_VALUE), + _n5entrycontrol("entry-unknown-converted", nothing, + MD.ConvertedType.T(99)), + ] + return controls +end + +function _n5generatedcontrol(name::String, node::N5Node, rows::Vector{Any}) + compiled = n5compile(node) + return N5SchemaControl(name, n5schema(node), n5shred(compiled, rows), + rows, :list, node) +end + +function n5listbindingcontrols() + rule2 = _n5rule2() + rule3 = _n5rule3() + rule2node = n5list("items", rule2.node.element; optional=true, + layout=:rule2, wrapper="array", annotation=:dual) + rows = Any[missing, [], Any[missing], Any[Int32(4), missing]] + optional = n5primitive("value", MD.Type.INT32; optional=true) + unannotatedschema = copy(rule3.schema) + unannotatedschema[3] = _n5element(unannotatedschema[3]; + logicalType=nothing, converted_type=nothing) + innervalue = n5list("array", n5primitive("array", MD.Type.INT32); + layout=:rule1) + innerstruct = n5struct("array", N5Node[innervalue]) + unannotatednode = n5list("items", innerstruct; optional=true, + layout=:rule2, wrapper="array") + unannotatedrows = Any[ + missing, + [], + Any[n5record(Any[])], + Any[n5record(Any[Int32(1), Int32(2)]), n5record(Any[]), + n5record(Any[Int32(3)])], + ] + controls = N5SchemaControl[ + _n5generatedcontrol("list-rule-2-multifield-array", rule2node, + rule2.rows), + N5SchemaControl("list-rule-3-repeated-array", rule3.schema, + rule3.streams, rule3.rows, :list, rule3.node), + N5SchemaControl("list-rule-3-unannotated-group", unannotatedschema, + rule3.streams, unannotatedrows, :list, unannotatednode), + _n5generatedcontrol("list-rule-5-Array", + n5list("items", optional; optional=true, layout=:rule5, + wrapper="Array", annotation=:dual), rows), + _n5generatedcontrol("list-rule-5-ARRAY", + n5list("items", optional; optional=true, layout=:rule5, + wrapper="ARRAY", annotation=:dual), rows), + _n5generatedcontrol("list-rule-5-case-mismatched-tuple", + n5list("items", optional; optional=true, layout=:rule5, + wrapper="Items_tuple", annotation=:dual), rows), + ] + blocked = _n5rule1() + schema = copy(blocked.schema) + schema[2] = _n5element(schema[2]; logicalType=_n5futurelogical(), + converted_type=MD.ConvertedType.LIST) + element = blocked.node.element + repeated = n5list(element.name, element; layout=:rule1) + semantic = n5struct(blocked.node.name, N5Node[repeated]; optional=true) + semanticrows = Any[ismissing(row) ? missing : n5record(row) + for row in blocked.rows] + push!(controls, N5SchemaControl("list-unknown-blocks-legacy", schema, + blocked.streams, semanticrows, :struct, semantic)) + return controls +end + +function _n5emptystreams(schema::Vector{MD.SchemaElement}) + return N5LeafStream[N5LeafStream(UInt64[], UInt64[], Any[], + leaf.max_repetition, leaf.max_definition) + for leaf in n5physicalleaves(schema)] +end + +function _n5schemafailure(name::String, schema::Vector{MD.SchemaElement}) + return N5SchemaFailure(name, schema, _n5emptystreams(schema)) +end + +function n5mapbindingfailures() + base = _n5standardmap().schema + failures = N5SchemaFailure[] + for (name, logical, converted) in ( + ("entry-logical-map", MD.LogicalType(MAP=MD.MapType()), nothing), + ("entry-logical-list", MD.LogicalType(LIST=MD.ListType()), nothing), + ("entry-logical-variant", _n5variantlogical(), nothing), + ("entry-logical-primitive", + MD.LogicalType(STRING=MD.StringType()), nothing), + ("entry-converted-map", nothing, MD.ConvertedType.MAP), + ("entry-converted-list", nothing, MD.ConvertedType.LIST), + ("entry-converted-primitive", nothing, MD.ConvertedType.UTF8)) + schema = copy(base) + schema[3] = _n5element(schema[3]; logicalType=logical, + converted_type=converted) + push!(failures, _n5schemafailure(name, schema)) + end + schema = copy(base) + schema[2] = _n5element(schema[2]; + logicalType=MD.LogicalType(STRING=MD.StringType()), + converted_type=MD.ConvertedType.MAP) + push!(failures, _n5schemafailure("outer-primitive-logical", schema)) + schema = copy(base) + schema[2] = _n5element(schema[2]; logicalType=nothing, + converted_type=MD.ConvertedType.UTF8) + push!(failures, _n5schemafailure("outer-primitive-converted", schema)) + schema = copy(base) + schema[2] = _n5element(schema[2]; + repetition_type=MD.FieldRepetitionType.REPEATED) + push!(failures, _n5schemafailure("outer-repeated-outside-list", schema)) + schema = MD.SchemaElement[base[1], _n5element(base[2]; num_children=Int32(0))] + push!(failures, _n5schemafailure("outer-zero-child", schema)) + extra = MD.SchemaElement(type_=MD.Type.INT32, + repetition_type=MD.FieldRepetitionType.REQUIRED, name="extra") + schema = MD.SchemaElement[base[1], + _n5element(base[2]; num_children=Int32(2)), base[3:5]..., extra] + push!(failures, _n5schemafailure("outer-two-children", schema)) + primitiveentry = MD.SchemaElement(type_=MD.Type.INT32, + repetition_type=MD.FieldRepetitionType.REPEATED, name="key_value") + schema = MD.SchemaElement[base[1], base[2], primitiveentry] + push!(failures, _n5schemafailure("entry-primitive", schema)) + schema = copy(base) + schema[3] = _n5element(schema[3]; + repetition_type=MD.FieldRepetitionType.REQUIRED) + push!(failures, _n5schemafailure("entry-non-repeated", schema)) + schema = MD.SchemaElement[base[1], base[2], + _n5element(base[3]; num_children=Int32(0))] + push!(failures, _n5schemafailure("entry-zero-child-key-absent", schema)) + schema = MD.SchemaElement[base[1], base[2], + _n5element(base[3]; num_children=Int32(3)), base[4], base[5], extra] + push!(failures, _n5schemafailure("entry-three-children", schema)) + schema = copy(base) + schema[4] = _n5element(schema[4]; + repetition_type=MD.FieldRepetitionType.REPEATED) + push!(failures, _n5schemafailure("key-repeated", schema)) + schema = copy(base) + schema[5] = _n5element(schema[5]; + repetition_type=MD.FieldRepetitionType.REPEATED) + push!(failures, _n5schemafailure("value-repeated", schema)) + return failures +end diff --git a/test/conformance/n5/model/integration.jl b/test/conformance/n5/model/integration.jl new file mode 100644 index 0000000..ab9da3d --- /dev/null +++ b/test/conformance/n5/model/integration.jl @@ -0,0 +1,304 @@ +function _n5normalize(node::N5Primitive, value) + value === missing && return missing + return value +end + +function _n5normalize(node::N5Struct, value) + value === missing && return missing + values = Any[] + sizehint!(values, length(node.fields)) + for (index, child) in enumerate(node.fields) + childvalue = try + value[child.name] + catch error + error isa MethodError || error isa KeyError || rethrow() + value[index] + end + push!(values, _n5normalize(child, childvalue)) + end + return N5Record(values) +end + +function _n5normalize(node::N5List, value) + value === missing && return missing + output = [] + sizehint!(output, length(value)) + for element in value + push!(output, _n5normalize(node.element, element)) + end + return output +end + +function _n5normalize(node::N5Map, value) + value === missing && return missing + entries = N5Entry[] + sizehint!(entries, length(value)) + for pair in value + key = _n5normalize(node.key, pair.first) + if node.value === nothing + push!(entries, N5Entry(key, nothing, false)) + else + push!(entries, N5Entry(key, + _n5normalize(node.value, pair.second), true)) + end + end + return N5MapValue(entries) +end + +function n5normalizetable(node::N5Node, table) + columns = values(table.columns) + column = first(columns) + output = [] + sizehint!(output, length(column)) + for value in column + push!(output, _n5normalize(node, value)) + end + return output +end + +function _n5samevalidity(expected::Nothing, actual) + return actual === nothing +end + +function _n5samevalidity(expected::BitVector, actual) + actual === nothing && return false + return expected == BitVector(actual) +end + +function _n5expectedleafbase(expected::N5ExpectedLeafVector) + expected.logical === :string && return String + expected.physical == MD.Type.BOOLEAN && return Bool + expected.physical == MD.Type.INT32 && return Int32 + expected.physical == MD.Type.INT64 && return Int64 + expected.physical == MD.Type.FLOAT && return Float32 + expected.physical == MD.Type.DOUBLE && return Float64 + expected.physical == MD.Type.BYTE_ARRAY && return Vector{UInt8} + expected.physical == MD.Type.FIXED_LEN_BYTE_ARRAY && return Vector{UInt8} + throw(ArgumentError("unsupported N5 expected leaf type $(expected.physical)")) +end + +function n5comparevectortree(expected::N5ExpectedLeafVector, actual) + if expected.physical == MD.Type.FIXED_LEN_BYTE_ARRAY + actual isa Parquet.FixedByteArrayVector || return false + else + actual isa Vector || return false + end + base = _n5expectedleafbase(expected) + expectedtype = expected.nullable ? Union{Missing,base} : base + eltype(actual) === expectedtype || return false + return isequal(expected.values, Any[actual...]) +end + +function n5comparevectortree(expected::N5ExpectedStructVector, actual) + actual isa Parquet.StructVector || return false + expected.names == actual.names || return false + expected.rows == actual.rows || return false + if expected.ranks === nothing + actual.ranks === nothing || return false + else + actual.ranks === nothing && return false + expected.ranks == Int[actual.ranks...] || return false + end + length(expected.children) == length(actual.children) || return false + for index in eachindex(expected.children, actual.children) + n5comparevectortree(expected.children[index], actual.children[index]) || + return false + end + return true +end + +function n5comparevectortree(expected::N5ExpectedListVector, actual) + actual isa Parquet.ListVector || return false + expected.offsets == Int[actual.offsets...] || return false + _n5samevalidity(expected.validity, actual.validity) || return false + return n5comparevectortree(expected.values, actual.values) +end + +function n5comparevectortree(expected::N5ExpectedMapVector, actual) + actual isa Parquet.MapVector || return false + expected.offsets == Int[actual.offsets...] || return false + _n5samevalidity(expected.validity, actual.validity) || return false + n5comparevectortree(expected.keys, actual.keys) || return false + if expected.values === nothing + return actual.values === nothing + end + actual.values === nothing && return false + return n5comparevectortree(expected.values, actual.values) +end + +function n5comparevectortree(expected::N5ExpectedTableVector, columns) + names = String[String(name) for name in keys(columns)] + names == expected.names || return false + actual = values(columns) + length(actual) == length(expected.children) || return false + for index in eachindex(expected.children) + length(actual[index]) == expected.rows || return false + n5comparevectortree(expected.children[index], actual[index]) || + return false + end + return true +end + +function n5productionencodedbytes(table, pageversion::Symbol; + codec::Symbol=:uncompressed) + options = (; checksum=false, dictionary=false, codec=codec, + pageversion=pageversion, encoding=:plain, rowgroupsize=nothing, + pagesize=nothing, pageindex=false, statistics=false) + privatefirst = Parquet._encodefile(table; options...) + privatesecond = Parquet._encodefile(table; options...) + firstio = IOBuffer() + Parquet.write(firstio, table; options...) + publicfirst = take!(firstio) + secondio = IOBuffer() + Parquet.write(secondio, table; options...) + publicsecond = take!(secondio) + return (; privatefirst, privatesecond, publicfirst, publicsecond) +end + +struct N5PropertyCodecFixture + caseid::Int + name::String + pageversion::Symbol + codec::Symbol + filename::String + sha256::String + bytes::Vector{UInt8} + schema::Vector{MD.SchemaElement} + paths::Vector{Vector{String}} + rows::Vector{Any} +end + +function n5propertyencodedbytes(case::N5PropertyCase, codec::Symbol) + codec in N5_PROPERTY_CODECS || throw(ArgumentError( + "unsupported N5 property codec $codec")) + source = n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=case.pageversion) + table = Parquet.Table(source) + try + return n5productionencodedbytes(table, case.pageversion; codec=codec) + finally + close(table) + end +end + +function n5propertycodecfixture(case::N5PropertyCase, codec::Symbol) + outputs = n5propertyencodedbytes(case, codec) + outputs.privatefirst == outputs.privatesecond == outputs.publicfirst == + outputs.publicsecond || throw(ArgumentError( + "N5 property codec output is not deterministic")) + bytes = outputs.publicfirst + source = n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=case.pageversion) + canonicalschema = n5decodefile(source).metadata.schema + filename = string(case.name, "-", case.pageversion, "-", codec, + ".parquet") + return N5PropertyCodecFixture(case.id, case.name, case.pageversion, codec, + filename, bytes2hex(SHA.sha256(bytes)), bytes, canonicalschema, + case.paths, case.rows) +end + +function n5propertycodecfixtures(cases::Vector{N5PropertyCase}) + fixtures = N5PropertyCodecFixture[] + sizehint!(fixtures, N5_PROPERTY_CODEC_COUNT * length(N5_PROPERTY_CODECS)) + for case in n5propertycodecsubset(cases) + for codec in N5_PROPERTY_CODECS + push!(fixtures, n5propertycodecfixture(case, codec)) + end + end + return fixtures +end + +function n5propertycodecfixtures() + cases, _ = n5propertycases() + return n5propertycodecfixtures(cases) +end + +function _n5metadataexact(left::TH.RawField, right::TH.RawField) + return left.id == right.id && left.type == right.type && + left.previd == right.previd && left.headerlength == right.headerlength && + left.bytes == right.bytes +end + +function _n5metadataexact(left::Tuple, right::Tuple) + length(left) == length(right) || return false + for index in eachindex(left, right) + _n5metadataexact(left[index], right[index]) || return false + end + return true +end + +function _n5metadataexact(left, right) + typeof(left) === typeof(right) || return false + T = typeof(left) + if isstructtype(T) && hasfield(T, :unknown_fields) + for index in 1:fieldcount(T) + _n5metadataexact(getfield(left, index), getfield(right, index)) || + return false + end + return true + end + return isequal(left, right) +end + +function n5schemaexact(left::Vector{MD.SchemaElement}, + right::Vector{MD.SchemaElement}) + length(left) == length(right) || return false + for index in eachindex(left, right) + _n5metadataexact(left[index], right[index]) || return false + end + return true +end + +function _n5productionlevels(levels, entries::Int) + levels === nothing && return fill(UInt64(0), entries) + return UInt64[levels...] +end + +function _n5physicalexpected(values::Vector{Any}, leaf::N5LeafSpec) + if leaf.logical === :string + return Any[Vector{UInt8}(codeunits(value)) for value in values] + end + return values +end + +function n5productionstreams(table, compiled::N5Compiled) + fields, rows = Parquet._writefields(table, Parquet.Limits()) + columns = Parquet.WriteColumn[] + for field in fields + append!(columns, field.leaves) + end + length(columns) == length(compiled.leaves) || throw(ArgumentError( + "production pre-encode leaf count differs from N5 model")) + streams = N5LeafStream[] + sizehint!(streams, length(columns)) + for (column, leaf) in zip(columns, compiled.leaves) + entries = column.repetitions === nothing ? + column.definitions === nothing ? length(column.values) : + length(column.definitions) : length(column.repetitions) + repetition = _n5productionlevels(column.repetitions, entries) + definition = _n5productionlevels(column.definitions, entries) + values = Any[column.values...] + push!(streams, N5LeafStream(repetition, definition, values, + Int(column.max_repetition_level), + Int(column.max_definition_level))) + end + return streams, fields, rows +end + +function n5compareproductionstreams(actual::Vector{N5LeafStream}, + expected::Vector{N5LeafStream}, compiled::N5Compiled) + length(actual) == length(expected) == length(compiled.leaves) || + return false + for index in eachindex(actual) + left = actual[index] + right = expected[index] + left.repetition == right.repetition || return false + left.definition == right.definition || return false + left.max_repetition == right.max_repetition || return false + left.max_definition == right.max_definition || return false + isequal(left.values, + _n5physicalexpected(right.values, compiled.leaves[index])) || + return false + end + return true +end diff --git a/test/conformance/n5/model/manifest.jl b/test/conformance/n5/model/manifest.jl new file mode 100644 index 0000000..2657b32 --- /dev/null +++ b/test/conformance/n5/model/manifest.jl @@ -0,0 +1,519 @@ +struct N5GoldenManifestEntry + name::String + schema::Vector{MD.SchemaElement} + paths::Vector{Vector{String}} + streams::Vector{N5LeafStream} + v1_sha256::String + v2_sha256::String +end + +const N5_GOLDEN_FILE_SHA256 = ( + ("list-rule-1", + "54a9e6c9bcb218b20cabfadbe8a5f54b5f247a363fd8f8a9d158f8abe8cbfc0f", + "bc5e6432b63dd5c3280fdaeed954005732d87164336b6a8e0da5ff78518ee354"), + ("list-rule-2", + "2992c963e21533f8a55ef2cc62e3187b40ca2ebf32321d677a394529238da861", + "4a46cc1bee0ee24f405fa5585606f90e4fb5b021f1886c8268270ff9e080cd52"), + ("list-rule-3", + "d4b62a49c69f89938dae4d02c31e725f8362fe70a8cb3571a92423f8a210a121", + "00f663c97d1f919ea16ee489ced5d53d23d8c0f8927351767ed43c4ecda3e15b"), + ("list-rule-4-array", + "95924bc48dd8b746a1bc18d1819f4e4a164f37a342744fe57a3ce935c818e488", + "e56cd1ec4bc6983c517c734c5e47486e4d491e010a8f2b631fe00d09f6df895d"), + ("list-rule-4-tuple", + "1fb94f3f7cb694f148bbc0cd6de380f979e2479c9b31482250da8986de191507", + "949a2102b8868e1feedf99657a8bcef7336b3025474ee67a464cdd0bb2a63a76"), + ("list-rule-5-required", + "b355239079e012cd40660be9daa98ffcb53bd0eb34fb9752988aa65bd0031a26", + "9c5173a38162e20ac8bd5bd6e416f02a5abe7839320f17f04b46d3b296169482"), + ("list-rule-5-paired", + "b9feac1fa757c89acb9b5169c63eba043d41614922d5fa3c024b75510eafde88", + "60d8739a9e86d4d0f1d048018d0f8490d5427cc47e335f33ec014455ec625dc7"), + ("list-rule-5-extended", + "590b16afb079231fda4193c6f5d921e74f1c9aee590de1422df04288aa111fca", + "34a9335bea7be8eb96f562e53753cbc8e37a93e1611e14a5950fbb195e6ead29"), + ("direct-list-of-map", + "b804532d75ab1b2de0715321d3c9006b625e8e22b479ae423b9656ba7a0598bc", + "df227a3c97364237a4a769e3ddf851943bf879e84df4a7d347c694c7722fa83e"), + ("map-standard", + "813baf63ad2d7db577c7f03a28e45fc9c337f30745f0b6c68e9b166724430cd2", + "951118322b172a4f705541e2c15db2cb0bcdf187994ca5f7b7426455d19af38e"), + ("map-key-only", + "8d4a2cd87b8c796252340035e38fd0031d954a97dc791d38b13ef2fe01e65aed", + "299e6c8ea9ff0727100640869286a656566cedf6d99d899fe45773d58b87e871"), + ("map-optional-key", + "9c7c47cebaae6a52117c6fe5b27a0be3184e99c9266e3d9dc5d65476def98c8b", + "77fab91a3365c936131bb9043524196f4ae7b838a90b473ce134c6673c3c6080"), +) + +const N5_GOLDEN_MANIFEST_SHA256 = + "185ad7b78e36b9e50128a96d81d0bf43bebce31cea4c4277fcdcee16214b31cd" +const N5_PROPERTY_MANIFEST_SHA256 = + "44d96748fb168f51343677f84097498626d8584a7d76fed2525ef6cbcf84ca3a" + +function _n5manifestinteger!(output::Vector{UInt8}, value::Integer) + negative = value < 0 + magnitude = negative ? -Int128(value) : Int128(value) + digits = string(magnitude) + length(digits) <= 39 || throw(ArgumentError( + "N5 manifest integer exceeds 39 digits")) + push!(output, negative ? UInt8('-') : UInt8('+')) + append!(output, codeunits(lpad(digits, 39, '0'))) + return +end + +function _n5manifestbytes!(output::Vector{UInt8}, value) + _n5manifestinteger!(output, length(value)) + append!(output, value) + return +end + +function _n5manifeststring!(output::Vector{UInt8}, value::AbstractString) + _n5manifestbytes!(output, codeunits(value)) + return +end + +function _n5manifestoptionalinteger!(output::Vector{UInt8}, value) + if value === nothing + push!(output, 0x00) + else + push!(output, 0x01) + _n5manifestinteger!(output, value) + end + return +end + +function _n5manifestoptionalenum!(output::Vector{UInt8}, value) + value === nothing && return _n5manifestoptionalinteger!(output, nothing) + return _n5manifestoptionalinteger!(output, value.value) +end + +function _n5manifestrawfields!(output::Vector{UInt8}, fields::Vector{TH.RawField}) + _n5manifestinteger!(output, length(fields)) + for field in fields + _n5manifestinteger!(output, field.id) + _n5manifestinteger!(output, field.type) + _n5manifestinteger!(output, field.previd) + _n5manifestinteger!(output, field.headerlength) + _n5manifestbytes!(output, field.bytes) + end + return +end + +function _n5manifestlogicalmember!(output::Vector{UInt8}, value) + value isa Union{MD.StringType,MD.MapType,MD.ListType} || + throw(ArgumentError("unsupported N5 manifest logical member $(typeof(value))")) + _n5manifestrawfields!(output, value.unknown_fields) + return +end + +function _n5manifestlogical!(output::Vector{UInt8}, logical::MD.LogicalType) + members = ( + (1, logical.STRING), + (2, logical.MAP), + (3, logical.LIST), + (4, logical.ENUM), + (5, logical.DECIMAL), + (6, logical.DATE), + (7, logical.TIME), + (8, logical.TIMESTAMP), + (10, logical.INTEGER), + (11, logical.UNKNOWN), + (12, logical.JSON), + (13, logical.BSON), + (14, logical.UUID), + (15, logical.FLOAT16), + (16, logical.VARIANT), + (17, logical.GEOMETRY), + (18, logical.GEOGRAPHY), + ) + for (id, member) in members + member === nothing && continue + _n5manifestinteger!(output, id) + _n5manifestlogicalmember!(output, member) + return + end + if isempty(logical.unknown_fields) + _n5manifestinteger!(output, 0) + else + _n5manifestinteger!(output, -1) + _n5manifestrawfields!(output, logical.unknown_fields) + end + return +end + +function _n5manifestoptionallogical!(output::Vector{UInt8}, logical) + if logical === nothing + push!(output, 0x00) + else + push!(output, 0x01) + _n5manifestlogical!(output, logical) + end + return +end + +function _n5manifestschemaelement!(output::Vector{UInt8}, element::MD.SchemaElement) + _n5manifestoptionalenum!(output, element.type_) + _n5manifestoptionalinteger!(output, element.type_length) + _n5manifestoptionalenum!(output, element.repetition_type) + _n5manifeststring!(output, element.name) + _n5manifestoptionalinteger!(output, element.num_children) + _n5manifestoptionalenum!(output, element.converted_type) + _n5manifestoptionalinteger!(output, element.scale) + _n5manifestoptionalinteger!(output, element.precision) + _n5manifestoptionalinteger!(output, element.field_id) + _n5manifestoptionallogical!(output, element.logicalType) + _n5manifestrawfields!(output, element.unknown_fields) + return +end + +function _n5manifestvalue!(output::Vector{UInt8}, value) + if value isa Int32 + push!(output, UInt8('i')) + _n5manifestinteger!(output, value) + elseif value isa Int64 + push!(output, UInt8('I')) + _n5manifestinteger!(output, value) + elseif value isa Float32 + push!(output, UInt8('f')) + _n5manifestinteger!(output, reinterpret(UInt32, value)) + elseif value isa Float64 + push!(output, UInt8('F')) + _n5manifestinteger!(output, reinterpret(UInt64, value)) + elseif value isa Bool + push!(output, value ? UInt8('t') : UInt8('b')) + elseif value isa AbstractString + push!(output, UInt8('s')) + _n5manifeststring!(output, value) + elseif value isa AbstractVector{UInt8} + push!(output, UInt8('x')) + _n5manifestbytes!(output, value) + else + throw(ArgumentError("unsupported N5 manifest value type $(typeof(value))")) + end + return +end + +function _n5manifeststream!(output::Vector{UInt8}, stream::N5LeafStream) + _n5manifestinteger!(output, stream.max_repetition) + _n5manifestinteger!(output, stream.max_definition) + _n5manifestinteger!(output, length(stream.repetition)) + for value in stream.repetition + _n5manifestinteger!(output, value) + end + _n5manifestinteger!(output, length(stream.definition)) + for value in stream.definition + _n5manifestinteger!(output, value) + end + _n5manifestinteger!(output, length(stream.values)) + for value in stream.values + _n5manifestvalue!(output, value) + end + return +end + +function n5goldenmanifest() + entries = N5GoldenManifestEntry[] + for case in n5bindinggoldens() + v1 = n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=:v1) + v2 = n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=:v2) + push!(entries, N5GoldenManifestEntry(case.name, case.schema, + case.paths, case.streams, bytes2hex(SHA.sha256(v1)), + bytes2hex(SHA.sha256(v2)))) + end + return entries +end + +function n5encodemanifest(entries::Vector{N5GoldenManifestEntry}) + output = UInt8[] + append!(output, codeunits("PARQUET-N5-A-MANIFEST-V2")) + _n5manifestinteger!(output, length(entries)) + for entry in entries + _n5manifeststring!(output, entry.name) + _n5manifestinteger!(output, length(entry.schema)) + for element in entry.schema + _n5manifestschemaelement!(output, element) + end + _n5manifestinteger!(output, length(entry.paths)) + for path in entry.paths + _n5manifestinteger!(output, length(path)) + for component in path + _n5manifeststring!(output, component) + end + end + _n5manifestinteger!(output, length(entry.streams)) + for stream in entry.streams + _n5manifeststream!(output, stream) + end + _n5manifeststring!(output, entry.v1_sha256) + _n5manifeststring!(output, entry.v2_sha256) + end + return output +end + +function n5manifestsha256(entries::Vector{N5GoldenManifestEntry}) + return bytes2hex(SHA.sha256(n5encodemanifest(entries))) +end + +function _n5propertynode!(output::Vector{UInt8}, node::N5Primitive) + push!(output, UInt8('P')) + _n5manifeststring!(output, node.name) + push!(output, node.optional ? 0x01 : 0x00) + _n5manifestinteger!(output, node.physical.value) + _n5manifeststring!(output, String(node.logical)) + return +end + +function _n5propertynode!(output::Vector{UInt8}, node::N5Struct) + push!(output, UInt8('S')) + _n5manifeststring!(output, node.name) + push!(output, node.optional ? 0x01 : 0x00) + _n5manifestinteger!(output, length(node.fields)) + for field in node.fields + _n5propertynode!(output, field) + end + return +end + +function _n5propertynode!(output::Vector{UInt8}, node::N5List) + push!(output, UInt8('L')) + _n5manifeststring!(output, node.name) + push!(output, node.optional ? 0x01 : 0x00) + _n5manifeststring!(output, String(node.layout)) + _n5manifeststring!(output, node.wrapper) + _n5manifeststring!(output, String(node.annotation)) + _n5propertynode!(output, node.element) + return +end + +function _n5propertynode!(output::Vector{UInt8}, node::N5Map) + push!(output, UInt8('M')) + _n5manifeststring!(output, node.name) + push!(output, node.optional ? 0x01 : 0x00) + _n5manifeststring!(output, String(node.marker)) + _n5manifeststring!(output, String(node.entrymarker)) + _n5manifeststring!(output, node.entryname) + _n5propertynode!(output, node.key) + if node.value === nothing + push!(output, 0x00) + else + push!(output, 0x01) + _n5propertynode!(output, node.value) + end + return +end + +function _n5propertysemantic!(output::Vector{UInt8}, value) + if ismissing(value) + push!(output, UInt8('m')) + elseif value isa N5Record + push!(output, UInt8('r')) + _n5manifestinteger!(output, length(value.values)) + for field in value.values + _n5propertysemantic!(output, field) + end + elseif value isa N5MapValue + push!(output, UInt8('M')) + _n5manifestinteger!(output, length(value.entries)) + for entry in value.entries + push!(output, entry.hasvalue ? 0x01 : 0x00) + _n5propertysemantic!(output, entry.key) + entry.hasvalue && _n5propertysemantic!(output, entry.value) + end + elseif value isa AbstractVector{UInt8} + _n5manifestvalue!(output, value) + elseif value isa AbstractVector + push!(output, UInt8('l')) + _n5manifestinteger!(output, length(value)) + for element in value + _n5propertysemantic!(output, element) + end + else + _n5manifestvalue!(output, value) + end + return +end + +function _n5propertyschema!(output::Vector{UInt8}, + schema::Vector{MD.SchemaElement}) + _n5manifestinteger!(output, length(schema)) + for element in schema + _n5manifestschemaelement!(output, element) + end + return +end + +function _n5propertypaths!(output::Vector{UInt8}, + paths::Vector{Vector{String}}) + _n5manifestinteger!(output, length(paths)) + for path in paths + _n5manifestinteger!(output, length(path)) + for component in path + _n5manifeststring!(output, component) + end + end + return +end + +function _n5propertystreams!(output::Vector{UInt8}, + streams::Vector{N5LeafStream}) + _n5manifestinteger!(output, length(streams)) + for stream in streams + _n5manifeststream!(output, stream) + end + return +end + +function _n5propertyrows!(output::Vector{UInt8}, rows::Vector{Any}) + _n5manifestinteger!(output, length(rows)) + for row in rows + _n5propertysemantic!(output, row) + end + return +end + +function _n5propertycount!(output::Vector{UInt8}, label::String, + counts, key) + _n5manifeststring!(output, label) + _n5manifestinteger!(output, get(() -> 0, counts, key)) + return +end + +function _n5propertycoverage!(output::Vector{UInt8}, + coverage::N5PropertyCoverage) + for kind in (:primitive, :struct, :list, :map) + _n5propertycount!(output, "node:$kind", coverage.nodes, kind) + _n5propertycount!(output, "recursive-node:$kind", + coverage.recursive_nodes, kind) + end + for layout in N5_PROPERTY_LIST_LAYOUTS + _n5propertycount!(output, "list-layout:$layout", + coverage.list_layouts, layout) + end + for annotation in N5_PROPERTY_LIST_ANNOTATIONS + _n5propertycount!(output, "list-annotation:$annotation", + coverage.list_annotations, annotation) + end + for marker in N5_PROPERTY_MAP_MARKERS + _n5propertycount!(output, "map-marker:$marker", + coverage.map_markers, marker) + end + for marker in N5_PROPERTY_ENTRY_MARKERS + _n5propertycount!(output, "entry-marker:$marker", + coverage.entry_markers, marker) + end + for repetition in (:required, :optional, :repeated) + _n5propertycount!(output, "repetition:$repetition", + coverage.repetitions, repetition) + end + for mode in (:absent, :required, :optional) + _n5propertycount!(output, "map-value:$mode", + coverage.value_modes, mode) + end + for mode in (:canonical, :arbitrary) + _n5propertycount!(output, "map-name:$mode", coverage.name_modes, mode) + end + for state in (:null, :empty, :present) + _n5propertycount!(output, "container:$state", + coverage.container_states, state) + end + for state in (:null, :present) + _n5propertycount!(output, "element:$state", + coverage.element_states, state) + _n5propertycount!(output, "value:$state", coverage.value_states, state) + end + for lengthvalue in 0:N5_PROPERTY_MAX_COLLECTION_LENGTH + _n5propertycount!(output, "collection:$lengthvalue", + coverage.collection_lengths, lengthvalue) + end + for depth in 1:6 + _n5propertycount!(output, "depth:$depth", coverage.depths, depth) + end + for width in 1:4 + _n5propertycount!(output, "width:$width", coverage.widths, width) + end + for rows in 0:24 + _n5propertycount!(output, "rows:$rows", coverage.row_counts, rows) + end + for pageversion in (:v1, :v2) + _n5propertycount!(output, "page:$pageversion", + coverage.page_versions, pageversion) + end + for physical in (MD.Type.INT32, MD.Type.INT64, MD.Type.FLOAT, + MD.Type.DOUBLE, MD.Type.BOOLEAN, MD.Type.BYTE_ARRAY) + _n5propertycount!(output, "physical:$(physical.value)", + coverage.physical_types, physical) + end + for (label, count) in (("duplicate-map", coverage.duplicate_maps), + ("complex-key", coverage.complex_keys), + ("complex-value", coverage.complex_values), + ("row-start", coverage.row_starts), + ("row-continuation", coverage.row_continuations), + ("rejected", coverage.rejected_candidates)) + _n5manifeststring!(output, label) + _n5manifestinteger!(output, count) + end + return +end + +function _n5propertycase!(output::Vector{UInt8}, case::N5PropertyCase) + _n5manifestinteger!(output, case.id) + _n5manifeststring!(output, case.name) + _n5manifeststring!(output, String(case.pageversion)) + for metric in (case.depth, case.width, case.astnodes, case.leaves, + case.levelentries, case.densevalues, case.payloadbytes) + _n5manifestinteger!(output, metric) + end + _n5propertynode!(output, case.node) + _n5propertyrows!(output, case.rows) + _n5propertyschema!(output, case.schema) + _n5propertypaths!(output, case.paths) + _n5propertystreams!(output, case.streams) + bytes = n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=case.pageversion) + _n5manifeststring!(output, bytes2hex(SHA.sha256(bytes))) + return +end + +function n5encodepropertymanifest(cases::Vector{N5PropertyCase}, rejected::Int) + length(cases) == N5_PROPERTY_CASE_COUNT || throw(ArgumentError( + "N5 property manifest needs exactly 256 cases")) + output = UInt8[] + append!(output, codeunits("PARQUET-N5-B-PROPERTY-MANIFEST-V1")) + _n5manifestinteger!(output, N5_PROPERTY_SEED) + _n5manifestinteger!(output, N5_PROPERTY_CANDIDATE_LAST) + _n5manifestinteger!(output, length(cases)) + for case in cases + _n5propertycase!(output, case) + end + coverage = n5propertycoverage(cases, rejected) + _n5propertycoverage!(output, coverage) + _n5manifestinteger!(output, length(N5_PROPERTY_CODEC_CASE_IDS)) + for id in N5_PROPERTY_CODEC_CASE_IDS + _n5manifestinteger!(output, id) + end + return output +end + +function n5propertymanifestsha256(cases::Vector{N5PropertyCase}, rejected::Int) + return bytes2hex(SHA.sha256(n5encodepropertymanifest(cases, rejected))) +end + +function _n5propertydiagnosticpart(writer, value) + output = UInt8[] + writer(output, value) + return bytes2hex(output) +end + +function n5propertydiagnostic(case::N5PropertyCase) + ast = _n5propertydiagnosticpart(_n5propertynode!, case.node) + schema = _n5propertydiagnosticpart(_n5propertyschema!, case.schema) + rows = _n5propertydiagnosticpart(_n5propertyrows!, case.rows) + streams = _n5propertydiagnosticpart(_n5propertystreams!, case.streams) + seed = string(N5_PROPERTY_SEED; base=16, pad=16) + return string("seed=0x", seed, " case=", case.id, + " ast=", ast, " schema=", schema, " rows=", rows, + " streams=", streams) +end diff --git a/test/conformance/n5/model/model.jl b/test/conformance/n5/model/model.jl new file mode 100644 index 0000000..b2784ca --- /dev/null +++ b/test/conformance/n5/model/model.jl @@ -0,0 +1,917 @@ +abstract type N5Node end + +struct N5Primitive <: N5Node + name::String + physical::MD.Type.T + optional::Bool + logical::Symbol +end + +struct N5Struct <: N5Node + name::String + optional::Bool + fields::Vector{N5Node} +end + +struct N5List <: N5Node + name::String + optional::Bool + element::N5Node + layout::Symbol + wrapper::String + annotation::Symbol +end + +struct N5Map <: N5Node + name::String + optional::Bool + key::N5Node + value::Union{Nothing,N5Node} + marker::Symbol + entrymarker::Symbol + entryname::String +end + +struct N5Record + values::Vector{Any} +end + +function n5record(values...) + return N5Record(Any[values...]) +end + +function Base.:(==)(left::N5Record, right::N5Record) + return left.values == right.values +end + +function Base.isequal(left::N5Record, right::N5Record) + return isequal(left.values, right.values) +end + +struct N5Entry + key::Any + value::Any + hasvalue::Bool +end + +function Base.:(==)(left::N5Entry, right::N5Entry) + return left.hasvalue == right.hasvalue && left.key == right.key && + left.value == right.value +end + +function Base.isequal(left::N5Entry, right::N5Entry) + return left.hasvalue == right.hasvalue && isequal(left.key, right.key) && + isequal(left.value, right.value) +end + +struct N5MapValue + entries::Vector{N5Entry} +end + +function n5mapvalue(pairs::Pair...) + return N5MapValue(N5Entry[N5Entry(pair.first, pair.second, true) + for pair in pairs]) +end + +function n5keyset(keys...) + return N5MapValue(N5Entry[N5Entry(key, nothing, false) for key in keys]) +end + +function n5maplookup(value::N5MapValue, key) + for index in length(value.entries):-1:1 + entry = value.entries[index] + isequal(entry.key, key) && return entry.hasvalue ? entry.value : missing + end + throw(KeyError(key)) +end + +function Base.:(==)(left::N5MapValue, right::N5MapValue) + return left.entries == right.entries +end + +function Base.isequal(left::N5MapValue, right::N5MapValue) + return isequal(left.entries, right.entries) +end + +abstract type N5PlanNode end + +struct N5LeafPlan <: N5PlanNode + node::N5Primitive + leaf::Int + parent_definition::Int + present_definition::Int + repetition_depth::Int +end + +struct N5StructPlan <: N5PlanNode + node::N5Struct + parent_definition::Int + present_definition::Int + repetition_depth::Int + children::Vector{N5PlanNode} + leaves::UnitRange{Int} +end + +struct N5ListPlan <: N5PlanNode + node::N5List + parent_definition::Int + present_definition::Int + entry_definition::Int + entry_repetition::Int + element::N5PlanNode + leaves::UnitRange{Int} +end + +struct N5MapPlan <: N5PlanNode + node::N5Map + parent_definition::Int + present_definition::Int + entry_definition::Int + entry_repetition::Int + key::N5PlanNode + value::Union{Nothing,N5PlanNode} + leaves::UnitRange{Int} +end + +struct N5LeafSpec + name::String + physical::MD.Type.T + logical::Symbol + max_repetition::Int + max_definition::Int +end + +struct N5Compiled + root::N5PlanNode + leaves::Vector{N5LeafSpec} +end + +struct N5LeafStream + repetition::Vector{UInt64} + definition::Vector{UInt64} + values::Vector{Any} + max_repetition::Int + max_definition::Int +end + +function Base.:(==)(left::N5LeafStream, right::N5LeafStream) + return left.repetition == right.repetition && + left.definition == right.definition && left.values == right.values && + left.max_repetition == right.max_repetition && + left.max_definition == right.max_definition +end + +function Base.isequal(left::N5LeafStream, right::N5LeafStream) + return isequal(left.repetition, right.repetition) && + isequal(left.definition, right.definition) && + isequal(left.values, right.values) && + left.max_repetition == right.max_repetition && + left.max_definition == right.max_definition +end + +mutable struct N5LeafBuilder + repetition::Vector{UInt64} + definition::Vector{UInt64} + values::Vector{Any} +end + +struct N5Absent end + +const N5_ABSENT = N5Absent() + +struct N5ExpandedLeaf + repetition::Vector{UInt64} + definition::Vector{UInt64} + entries::Vector{Any} + max_repetition::Int + max_definition::Int +end + +struct N5PhysicalLeaf + element::MD.SchemaElement + path::Vector{String} + max_repetition::Int + max_definition::Int +end + +abstract type N5ExpectedVector end + +struct N5ExpectedLeafVector <: N5ExpectedVector + physical::MD.Type.T + logical::Symbol + nullable::Bool + values::Vector{Any} +end + +struct N5ExpectedStructVector <: N5ExpectedVector + names::Vector{String} + ranks::Union{Nothing,Vector{Int}} + children::Vector{N5ExpectedVector} + rows::Int +end + +struct N5ExpectedListVector <: N5ExpectedVector + offsets::Vector{Int} + validity::Union{Nothing,BitVector} + values::N5ExpectedVector +end + +struct N5ExpectedMapVector <: N5ExpectedVector + offsets::Vector{Int} + validity::Union{Nothing,BitVector} + keys::N5ExpectedVector + values::Union{Nothing,N5ExpectedVector} +end + +struct N5ExpectedTableVector + names::Vector{String} + children::Vector{N5ExpectedVector} + rows::Int +end + +function n5primitive(name::AbstractString, physical::MD.Type.T; + optional::Bool=false, logical::Symbol=:none) + logical in (:none, :string) || throw(ArgumentError( + "unsupported N5 primitive logical annotation $logical")) + logical === :string && physical != MD.Type.BYTE_ARRAY && throw(ArgumentError( + "N5 STRING primitive must use BYTE_ARRAY")) + return N5Primitive(String(name), physical, optional, logical) +end + +function n5struct(name::AbstractString, fields::Vector{N5Node}; + optional::Bool=false) + isempty(fields) && throw(ArgumentError("N5 struct must have at least one field")) + return N5Struct(String(name), optional, fields) +end + +function n5list(name::AbstractString, element::N5Node; optional::Bool=false, + layout::Symbol=:canonical, wrapper::AbstractString="list", + annotation::Symbol=:dual) + layout in (:canonical, :rule1, :rule2, :rule3, :rule4, :rule5) || + throw(ArgumentError("unsupported N5 LIST layout $layout")) + annotation in (:modern, :legacy, :dual, :conflict) || + throw(ArgumentError("unsupported N5 LIST annotation $annotation")) + layout === :rule1 && (!(element isa N5Primitive) || element.optional) && + throw(ArgumentError("LIST rule 1 needs a required primitive element")) + layout === :rule2 && !(element isa N5Struct) && + throw(ArgumentError("LIST rule 2 needs a struct element")) + layout === :rule3 && !(element isa Union{N5List,N5Map}) && + throw(ArgumentError("LIST rule 3 needs a repeated collection child")) + layout === :rule4 && (!(element isa N5Struct) || length(element.fields) != 1) && + throw(ArgumentError("LIST rule 4 needs a one-field struct element")) + return N5List(String(name), optional, element, layout, String(wrapper), annotation) +end + +function n5map(name::AbstractString, key::N5Node, + value::Union{Nothing,N5Node}; optional::Bool=false, + marker::Symbol=:modern, entrymarker::Symbol=:none, + entryname::AbstractString="key_value") + marker in (:modern, :legacy, :alias, :dual, :conflict, + :modern_alias, :modern_primitive, :modern_unknown) || + throw(ArgumentError("unsupported N5 MAP marker $marker")) + entrymarker in (:none, :marked, :empty, :future, :future_marked, + :unknown_converted) || + throw(ArgumentError("unsupported N5 MAP entry marker $entrymarker")) + return N5Map(String(name), optional, key, value, marker, entrymarker, + String(entryname)) +end + +function _n5expectedvector(node::N5Primitive, values::Vector{Any}; + force_required::Bool=false) + for value in values + ismissing(value) && (!node.optional || force_required) && throw(ArgumentError( + "required N5 vector leaf $(repr(node.name)) is null")) + end + return N5ExpectedLeafVector(node.physical, node.logical, + node.optional && !force_required, copy(values)) +end + +function _n5expectedvector(node::N5Struct, values::Vector{Any}; + force_required::Bool=false) + required = force_required || !node.optional + ranks = required ? nothing : Int[0] + childvalues = [Any[] for _ in node.fields] + present = 0 + for value in values + if ismissing(value) + required && throw(ArgumentError( + "required N5 vector struct $(repr(node.name)) is null")) + else + value isa N5Record || throw(ArgumentError( + "N5 vector struct $(repr(node.name)) needs an N5Record")) + length(value.values) == length(node.fields) || throw(ArgumentError( + "N5 vector struct $(repr(node.name)) has the wrong field count")) + for index in eachindex(node.fields) + push!(childvalues[index], value.values[index]) + end + present += 1 + end + ranks === nothing || push!(ranks, present) + end + children = N5ExpectedVector[_n5expectedvector(field, childvalues[index]) + for (index, field) in enumerate(node.fields)] + return N5ExpectedStructVector(String[field.name for field in node.fields], + ranks, children, length(values)) +end + +function _n5expectedvector(node::N5List, values::Vector{Any}; + force_required::Bool=false) + required = force_required || !node.optional + offsets = Int[0] + validity = required ? nothing : BitVector() + elements = Any[] + for value in values + if ismissing(value) + required && throw(ArgumentError( + "required N5 vector list $(repr(node.name)) is null")) + push!(validity, false) + else + value isa AbstractVector || throw(ArgumentError( + "N5 vector list $(repr(node.name)) needs a vector")) + validity === nothing || push!(validity, true) + append!(elements, value) + end + push!(offsets, length(elements)) + end + child = _n5expectedvector(node.element, elements) + return N5ExpectedListVector(offsets, validity, child) +end + +function _n5expectedvector(node::N5Map, values::Vector{Any}; + force_required::Bool=false) + required = force_required || !node.optional + offsets = Int[0] + validity = required ? nothing : BitVector() + keys = Any[] + mapvalues = Any[] + for value in values + if ismissing(value) + required && throw(ArgumentError( + "required N5 vector map $(repr(node.name)) is null")) + push!(validity, false) + else + value isa N5MapValue || throw(ArgumentError( + "N5 vector map $(repr(node.name)) needs an N5MapValue")) + validity === nothing || push!(validity, true) + for entry in value.entries + ismissing(entry.key) && throw(ArgumentError( + "N5 vector map $(repr(node.name)) has a null key")) + push!(keys, entry.key) + node.value === nothing || begin + entry.hasvalue || throw(ArgumentError( + "N5 vector map $(repr(node.name)) is missing a value field")) + push!(mapvalues, entry.value) + end + end + end + push!(offsets, length(keys)) + end + key = _n5expectedvector(node.key, keys; force_required=true) + value = node.value === nothing ? nothing : + _n5expectedvector(node.value, mapvalues) + return N5ExpectedMapVector(offsets, validity, key, value) +end + +function n5expectedvectortree(node::N5Node, rows::AbstractVector) + values = Any[rows...] + child = _n5expectedvector(node, values) + return N5ExpectedTableVector(String[node.name], N5ExpectedVector[child], + length(values)) +end + +function _n5range(firstleaf::Int, leaves::Vector{N5LeafSpec}) + return firstleaf:length(leaves) +end + +function _n5compile!(leaves::Vector{N5LeafSpec}, node::N5Primitive, + definition::Int, repetition::Int) + parent = definition + present = definition + Int(node.optional) + push!(leaves, N5LeafSpec(node.name, node.physical, node.logical, + repetition, present)) + return N5LeafPlan(node, length(leaves), parent, present, repetition) +end + +function _n5compile!(leaves::Vector{N5LeafSpec}, node::N5Struct, + definition::Int, repetition::Int) + firstleaf = length(leaves) + 1 + parent = definition + present = definition + Int(node.optional) + children = N5PlanNode[] + sizehint!(children, length(node.fields)) + for field in node.fields + push!(children, _n5compile!(leaves, field, present, repetition)) + end + return N5StructPlan(node, parent, present, repetition, children, + _n5range(firstleaf, leaves)) +end + +function _n5compile!(leaves::Vector{N5LeafSpec}, node::N5List, + definition::Int, repetition::Int) + firstleaf = length(leaves) + 1 + parent = definition + present = definition + Int(node.optional) + entrydefinition = present + 1 + entryrepetition = repetition + 1 + element = _n5compile!(leaves, node.element, entrydefinition, + entryrepetition) + return N5ListPlan(node, parent, present, entrydefinition, + entryrepetition, element, _n5range(firstleaf, leaves)) +end + +function _n5compile!(leaves::Vector{N5LeafSpec}, node::N5Map, + definition::Int, repetition::Int) + firstleaf = length(leaves) + 1 + parent = definition + present = definition + Int(node.optional) + entrydefinition = present + 1 + entryrepetition = repetition + 1 + key = _n5compile!(leaves, node.key, entrydefinition, entryrepetition) + value = node.value === nothing ? nothing : _n5compile!(leaves, + node.value, entrydefinition, entryrepetition) + return N5MapPlan(node, parent, present, entrydefinition, + entryrepetition, key, value, _n5range(firstleaf, leaves)) +end + +function n5compile(node::N5Node) + leaves = N5LeafSpec[] + root = _n5compile!(leaves, node, 0, 0) + return N5Compiled(root, leaves) +end + +function _n5planleaves(plan::N5LeafPlan) + return plan.leaf:plan.leaf +end + +function _n5planleaves(plan::Union{N5StructPlan,N5ListPlan,N5MapPlan}) + return plan.leaves +end + +function _n5emitnull!(builders::Vector{N5LeafBuilder}, plan::N5PlanNode, + repetition::Int, definition::Int) + for leaf in _n5planleaves(plan) + push!(builders[leaf].repetition, UInt64(repetition)) + push!(builders[leaf].definition, UInt64(definition)) + end + return +end + +function _n5shred!(builders::Vector{N5LeafBuilder}, plan::N5LeafPlan, + value, repetition::Int) + builder = builders[plan.leaf] + push!(builder.repetition, UInt64(repetition)) + if value === missing + plan.node.optional || throw(ArgumentError( + "required N5 primitive $(repr(plan.node.name)) is null")) + push!(builder.definition, UInt64(plan.parent_definition)) + return + end + push!(builder.definition, UInt64(plan.present_definition)) + push!(builder.values, value) + return +end + +function _n5shred!(builders::Vector{N5LeafBuilder}, plan::N5StructPlan, + value, repetition::Int) + if value === missing + plan.node.optional || throw(ArgumentError( + "required N5 struct $(repr(plan.node.name)) is null")) + _n5emitnull!(builders, plan, repetition, plan.parent_definition) + return + end + value isa N5Record || throw(ArgumentError( + "N5 struct $(repr(plan.node.name)) needs an N5Record")) + length(value.values) == length(plan.children) || throw(ArgumentError( + "N5 struct $(repr(plan.node.name)) has the wrong field count")) + for (child, fieldvalue) in zip(plan.children, value.values) + _n5shred!(builders, child, fieldvalue, repetition) + end + return +end + +function _n5shred!(builders::Vector{N5LeafBuilder}, plan::N5ListPlan, + value, repetition::Int) + if value === missing + plan.node.optional || throw(ArgumentError( + "required N5 LIST $(repr(plan.node.name)) is null")) + _n5emitnull!(builders, plan, repetition, plan.parent_definition) + return + end + value isa AbstractVector || throw(ArgumentError( + "N5 LIST $(repr(plan.node.name)) needs a vector")) + if isempty(value) + _n5emitnull!(builders, plan, repetition, plan.present_definition) + return + end + for index in eachindex(value) + entryrepetition = index == firstindex(value) ? repetition : + plan.entry_repetition + _n5shred!(builders, plan.element, value[index], entryrepetition) + end + return +end + +function _n5shred!(builders::Vector{N5LeafBuilder}, plan::N5MapPlan, + value, repetition::Int) + if value === missing + plan.node.optional || throw(ArgumentError( + "required N5 MAP $(repr(plan.node.name)) is null")) + _n5emitnull!(builders, plan, repetition, plan.parent_definition) + return + end + value isa N5MapValue || throw(ArgumentError( + "N5 MAP $(repr(plan.node.name)) needs an N5MapValue")) + if isempty(value.entries) + _n5emitnull!(builders, plan, repetition, plan.present_definition) + return + end + for index in eachindex(value.entries) + entry = value.entries[index] + entry.key === missing && throw(ArgumentError("N5 MAP key is null")) + entryrepetition = index == firstindex(value.entries) ? repetition : + plan.entry_repetition + _n5shred!(builders, plan.key, entry.key, entryrepetition) + if plan.value === nothing + entry.hasvalue && throw(ArgumentError( + "key-only N5 MAP entry carries a value")) + else + entry.hasvalue || throw(ArgumentError("N5 MAP entry omits its value")) + _n5shred!(builders, plan.value, entry.value, entryrepetition) + end + end + return +end + +function n5shred(compiled::N5Compiled, rows::AbstractVector) + builders = N5LeafBuilder[N5LeafBuilder(UInt64[], UInt64[], []) + for _ in compiled.leaves] + for row in rows + _n5shred!(builders, compiled.root, row, 0) + end + streams = N5LeafStream[] + sizehint!(streams, length(builders)) + for (builder, leaf) in zip(builders, compiled.leaves) + push!(streams, N5LeafStream(builder.repetition, builder.definition, + builder.values, leaf.max_repetition, leaf.max_definition)) + end + return streams +end + +function _n5expand(stream::N5LeafStream) + length(stream.repetition) == length(stream.definition) || + throw(ArgumentError("N5 stream level lengths differ")) + entries = Vector{Any}(undef, length(stream.definition)) + fill!(entries, N5_ABSENT) + dense = 1 + for index in eachindex(stream.definition) + if stream.definition[index] == UInt64(stream.max_definition) + dense <= length(stream.values) || throw(ArgumentError( + "N5 stream dense values underflow")) + entries[index] = stream.values[dense] + dense += 1 + end + end + dense == length(stream.values) + 1 || throw(ArgumentError( + "N5 stream dense values overflow")) + return N5ExpandedLeaf(stream.repetition, stream.definition, entries, + stream.max_repetition, stream.max_definition) +end + +function _n5rowranges(stream::N5ExpandedLeaf, rows::Int) + rows == 0 && return UnitRange{Int}[] + starts = Int[] + for index in eachindex(stream.repetition) + iszero(stream.repetition[index]) && push!(starts, index) + end + length(starts) == rows || throw(ArgumentError( + "N5 stream has $(length(starts)) rows, expected $rows")) + ranges = UnitRange{Int}[] + sizehint!(ranges, rows) + for index in eachindex(starts) + stop = index == length(starts) ? length(stream.repetition) : + starts[index + 1] - 1 + push!(ranges, starts[index]:stop) + end + return ranges +end + +function _n5driver(plan::N5PlanNode) + return first(_n5planleaves(plan)) +end + +function _n5firstdefinition(streams::Vector{N5ExpandedLeaf}, + ranges::Vector{UnitRange{Int}}, plan::N5PlanNode) + leaf = _n5driver(plan) + range = ranges[leaf] + isempty(range) && throw(ArgumentError("N5 assembler received an empty range")) + return Int(streams[leaf].definition[first(range)]) +end + +function _n5partitions(stream::N5ExpandedLeaf, range::UnitRange{Int}, + repetition::Int) + isempty(range) && throw(ArgumentError("N5 assembler cannot partition an empty range")) + starts = Int[first(range)] + for index in (first(range) + 1):last(range) + stream.repetition[index] <= UInt64(repetition) && push!(starts, index) + end + ranges = UnitRange{Int}[] + sizehint!(ranges, length(starts)) + for index in eachindex(starts) + stop = index == length(starts) ? last(range) : starts[index + 1] - 1 + push!(ranges, starts[index]:stop) + end + return ranges +end + +function _n5entryranges(streams::Vector{N5ExpandedLeaf}, + ranges::Vector{UnitRange{Int}}, plan::N5PlanNode, repetition::Int) + leaves = _n5planleaves(plan) + partitions = Vector{Vector{UnitRange{Int}}}(undef, length(leaves)) + count = 0 + for (slot, leaf) in enumerate(leaves) + current = _n5partitions(streams[leaf], ranges[leaf], repetition) + if slot == 1 + count = length(current) + else + length(current) == count || throw(ArgumentError( + "N5 sibling occurrence counts differ")) + end + partitions[slot] = current + end + output = Vector{Vector{UnitRange{Int}}}(undef, count) + for occurrence in 1:count + current = copy(ranges) + for (slot, leaf) in enumerate(leaves) + current[leaf] = partitions[slot][occurrence] + end + output[occurrence] = current + end + return output +end + +function _n5assemble(plan::N5LeafPlan, streams::Vector{N5ExpandedLeaf}, + ranges::Vector{UnitRange{Int}}) + range = ranges[plan.leaf] + length(range) == 1 || throw(ArgumentError( + "N5 primitive occurrence has $(length(range)) level entries")) + index = first(range) + definition = Int(streams[plan.leaf].definition[index]) + if definition < plan.present_definition + plan.node.optional || throw(ArgumentError( + "required N5 primitive is absent")) + return missing + end + value = streams[plan.leaf].entries[index] + value isa N5Absent && throw(ArgumentError("N5 primitive dense value is absent")) + return value +end + +function _n5assemble(plan::N5StructPlan, streams::Vector{N5ExpandedLeaf}, + ranges::Vector{UnitRange{Int}}) + definition = _n5firstdefinition(streams, ranges, plan) + if definition < plan.present_definition + plan.node.optional || throw(ArgumentError("required N5 struct is absent")) + return missing + end + values = Any[_n5assemble(child, streams, ranges) for child in plan.children] + return N5Record(values) +end + +function _n5assemble(plan::N5ListPlan, streams::Vector{N5ExpandedLeaf}, + ranges::Vector{UnitRange{Int}}) + definition = _n5firstdefinition(streams, ranges, plan) + if definition < plan.present_definition + plan.node.optional || throw(ArgumentError("required N5 LIST is absent")) + return missing + end + definition < plan.entry_definition && return [] + entries = _n5entryranges(streams, ranges, plan, + plan.entry_repetition) + return Any[_n5assemble(plan.element, streams, entry) for entry in entries] +end + +function _n5assemble(plan::N5MapPlan, streams::Vector{N5ExpandedLeaf}, + ranges::Vector{UnitRange{Int}}) + definition = _n5firstdefinition(streams, ranges, plan) + if definition < plan.present_definition + plan.node.optional || throw(ArgumentError("required N5 MAP is absent")) + return missing + end + definition < plan.entry_definition && return N5MapValue(N5Entry[]) + rangesbyentry = _n5entryranges(streams, ranges, plan, + plan.entry_repetition) + entries = N5Entry[] + sizehint!(entries, length(rangesbyentry)) + for entryranges in rangesbyentry + key = _n5assemble(plan.key, streams, entryranges) + key === missing && throw(ArgumentError("N5 MAP key is null")) + if plan.value === nothing + push!(entries, N5Entry(key, nothing, false)) + else + value = _n5assemble(plan.value, streams, entryranges) + push!(entries, N5Entry(key, value, true)) + end + end + return N5MapValue(entries) +end + +function n5assemble(compiled::N5Compiled, streams::Vector{N5LeafStream}, + rows::Integer) + length(streams) == length(compiled.leaves) || throw(ArgumentError( + "N5 assembler leaf count differs")) + rowcount = Int(rows) + rowcount >= 0 || throw(ArgumentError("N5 row count is negative")) + expanded = N5ExpandedLeaf[_n5expand(stream) for stream in streams] + ranges = Vector{Vector{UnitRange{Int}}}(undef, length(expanded)) + for index in eachindex(expanded) + ranges[index] = _n5rowranges(expanded[index], rowcount) + end + output = [] + sizehint!(output, rowcount) + for row in 1:rowcount + rowranges = UnitRange{Int}[ranges[leaf][row] for leaf in eachindex(ranges)] + push!(output, _n5assemble(compiled.root, expanded, rowranges)) + end + return output +end + +function _n5repetition(optional::Bool) + return optional ? MD.FieldRepetitionType.OPTIONAL : + MD.FieldRepetitionType.REQUIRED +end + +function _n5repetition(optional::Bool, override::Union{Nothing,Symbol}) + override === nothing && return _n5repetition(optional) + override === :repeated && return MD.FieldRepetitionType.REPEATED + throw(ArgumentError("unsupported N5 repetition override $override")) +end + +function _n5primitiveannotations(node::N5Primitive) + node.logical === :none && return (nothing, nothing) + logical = MD.LogicalType(STRING=MD.StringType()) + return (logical, MD.ConvertedType.UTF8) +end + +function _n5listannotations(annotation::Symbol) + logical = annotation in (:modern, :dual, :conflict) ? + MD.LogicalType(LIST=MD.ListType()) : nothing + converted = annotation in (:legacy, :dual) ? MD.ConvertedType.LIST : + annotation === :conflict ? MD.ConvertedType.MAP : nothing + return (logical, converted) +end + +function _n5mapannotations(marker::Symbol) + logical = marker in (:modern, :dual, :conflict, :modern_alias, + :modern_primitive, :modern_unknown) ? + MD.LogicalType(MAP=MD.MapType()) : nothing + converted = marker in (:legacy, :dual) ? MD.ConvertedType.MAP : + marker === :alias ? MD.ConvertedType.MAP_KEY_VALUE : + marker === :conflict ? MD.ConvertedType.LIST : + marker === :modern_alias ? MD.ConvertedType.MAP_KEY_VALUE : + marker === :modern_primitive ? MD.ConvertedType.UTF8 : + marker === :modern_unknown ? MD.ConvertedType.T(99) : nothing + return (logical, converted) +end + +function _n5futurelogical(id::Int16=Int16(2555)) + return MD.LogicalType(unknown_fields=( + TH.RawField(id, TH.STRUCT, UInt8[0x00]),)) +end + +function _n5mapentryannotations(marker::Symbol) + marker === :none && return (nothing, nothing) + marker === :marked && return (nothing, MD.ConvertedType.MAP_KEY_VALUE) + marker === :empty && return (MD.LogicalType(), nothing) + marker === :future && return (_n5futurelogical(), nothing) + marker === :future_marked && return (_n5futurelogical(), + MD.ConvertedType.MAP_KEY_VALUE) + marker === :unknown_converted && return (nothing, MD.ConvertedType.T(99)) + throw(ArgumentError("unsupported N5 MAP entry marker $marker")) +end + +function _n5schemafield!(schema::Vector{MD.SchemaElement}, node::N5Primitive; + repetition::Union{Nothing,Symbol}=nothing, + name::Union{Nothing,String}=nothing) + logical, converted = _n5primitiveannotations(node) + push!(schema, MD.SchemaElement(type_=node.physical, + repetition_type=_n5repetition(node.optional, repetition), + name=something(name, node.name), converted_type=converted, + logicalType=logical)) + return +end + +function _n5schemafield!(schema::Vector{MD.SchemaElement}, node::N5Struct; + repetition::Union{Nothing,Symbol}=nothing, + name::Union{Nothing,String}=nothing) + push!(schema, MD.SchemaElement( + repetition_type=_n5repetition(node.optional, repetition), + name=something(name, node.name), num_children=Int32(length(node.fields)))) + for field in node.fields + _n5schemafield!(schema, field) + end + return +end + +function _n5schemalistentry!(schema::Vector{MD.SchemaElement}, node::N5List) + if node.layout === :rule1 + _n5schemafield!(schema, node.element; repetition=:repeated) + elseif node.layout === :rule2 + element = node.element::N5Struct + _n5schemafield!(schema, element; repetition=:repeated, + name=node.wrapper) + elseif node.layout === :rule3 + _n5schemafield!(schema, node.element; repetition=:repeated, + name=node.wrapper) + elseif node.layout === :rule4 + element = node.element::N5Struct + _n5schemafield!(schema, element; repetition=:repeated, + name=node.wrapper) + elseif node.layout === :rule5 + push!(schema, MD.SchemaElement( + repetition_type=MD.FieldRepetitionType.REPEATED, + name=node.wrapper, num_children=Int32(1))) + _n5schemafield!(schema, node.element) + else + push!(schema, MD.SchemaElement( + repetition_type=MD.FieldRepetitionType.REPEATED, + name=node.wrapper, num_children=Int32(1))) + _n5schemafield!(schema, node.element; name="element") + end + return +end + +function _n5schemafield!(schema::Vector{MD.SchemaElement}, node::N5List; + repetition::Union{Nothing,Symbol}=nothing, + name::Union{Nothing,String}=nothing) + logical, converted = _n5listannotations(node.annotation) + push!(schema, MD.SchemaElement( + repetition_type=_n5repetition(node.optional, repetition), + name=something(name, node.name), num_children=Int32(1), + converted_type=converted, logicalType=logical)) + _n5schemalistentry!(schema, node) + return +end + +function _n5schemafield!(schema::Vector{MD.SchemaElement}, node::N5Map; + repetition::Union{Nothing,Symbol}=nothing, + name::Union{Nothing,String}=nothing) + logical, converted = _n5mapannotations(node.marker) + push!(schema, MD.SchemaElement( + repetition_type=_n5repetition(node.optional, repetition), + name=something(name, node.name), num_children=Int32(1), + converted_type=converted, logicalType=logical)) + childcount = node.value === nothing ? 1 : 2 + entrylogical, entryconverted = _n5mapentryannotations(node.entrymarker) + push!(schema, MD.SchemaElement( + repetition_type=MD.FieldRepetitionType.REPEATED, + name=node.entryname, num_children=Int32(childcount), + converted_type=entryconverted, logicalType=entrylogical)) + _n5schemafield!(schema, node.key) + node.value === nothing || _n5schemafield!(schema, node.value) + return +end + +function n5schema(node::N5Node; rootname::AbstractString="schema") + schema = MD.SchemaElement[MD.SchemaElement(name=String(rootname), + num_children=Int32(1))] + _n5schemafield!(schema, node) + return schema +end + +function _n5physicalnode!(leaves::Vector{N5PhysicalLeaf}, + schema::Vector{MD.SchemaElement}, index::Int, path::Vector{String}, + repetition::Int, definition::Int) + index <= length(schema) || throw(ArgumentError("N5 schema ends early")) + element = schema[index] + nextpath = copy(path) + push!(nextpath, element.name) + fieldrepetition = element.repetition_type + nextrepetition = repetition + Int(fieldrepetition == + MD.FieldRepetitionType.REPEATED) + nextdefinition = definition + Int(fieldrepetition in + (MD.FieldRepetitionType.OPTIONAL, MD.FieldRepetitionType.REPEATED)) + if element.type_ !== nothing + push!(leaves, N5PhysicalLeaf(element, nextpath, nextrepetition, + nextdefinition)) + return index + 1 + end + children = something(element.num_children, Int32(0)) + children >= 0 || throw(ArgumentError("N5 schema has negative child count")) + cursor = index + 1 + for _ in 1:Int(children) + cursor = _n5physicalnode!(leaves, schema, cursor, nextpath, + nextrepetition, nextdefinition) + end + return cursor +end + +function n5physicalleaves(schema::Vector{MD.SchemaElement}) + isempty(schema) && throw(ArgumentError("N5 schema is empty")) + root = schema[1] + root.type_ === nothing || throw(ArgumentError("N5 root is primitive")) + root.num_children == 1 || throw(ArgumentError("N5 model needs one root field")) + leaves = N5PhysicalLeaf[] + cursor = _n5physicalnode!(leaves, schema, 2, String[], 0, 0) + cursor == length(schema) + 1 || throw(ArgumentError("N5 schema has trailing nodes")) + return leaves +end diff --git a/test/conformance/n5/model/properties.jl b/test/conformance/n5/model/properties.jl new file mode 100644 index 0000000..8afed04 --- /dev/null +++ b/test/conformance/n5/model/properties.jl @@ -0,0 +1,1039 @@ +const N5_PROPERTY_SEED = UInt64(0x4e355f4c49535435) +const N5_PROPERTY_CANDIDATE_LAST = 4095 +const N5_PROPERTY_CASE_COUNT = 256 +const N5_PROPERTY_CODEC_COUNT = 32 +const N5_PROPERTY_MAX_AST_NODES = 64 +const N5_PROPERTY_MAX_LEAVES = 16 +const N5_PROPERTY_MAX_LEVEL_ENTRIES = 2048 +const N5_PROPERTY_MAX_DENSE_VALUES = 2048 +const N5_PROPERTY_MAX_PAYLOAD_BYTES = 256 * 1024 +const N5_PROPERTY_MAX_COLLECTION_LENGTH = 5 +const N5_PROPERTY_LIST_LAYOUTS = + (:canonical, :rule1, :rule2, :rule3, :rule4, :rule5) +const N5_PROPERTY_LIST_ANNOTATIONS = (:modern, :legacy, :dual, :conflict) +const N5_PROPERTY_MAP_MARKERS = (:modern, :dual, :modern_alias, :conflict, + :modern_primitive, :modern_unknown, :legacy, :alias) +const N5_PROPERTY_ENTRY_MARKERS = (:none, :marked, :empty, :future, + :future_marked, :unknown_converted) +const N5_PROPERTY_CODECS = + (:uncompressed, :snappy, :gzip, :brotli, :zstd, :lz4_raw) +const N5_PROPERTY_CODEC_CASE_IDS = ( + 2, 4, 5, 6, 7, 8, 9, 10, + 11, 14, 15, 18, 28, 34, 38, 48, + 58, 62, 96, 106, 107, 117, 122, 123, + 144, 145, 148, 178, 191, 202, 249, 251, +) +const N5_PROPERTY_REJECTED_PREFIX = ( + (29, :depth), + (63, :width), + (95, :astnodes), + (154, :leaves), + (201, :levelentries), + (254, :payloadbytes), +) + +mutable struct N5SplitMix64 + state::UInt64 +end + +struct N5PropertyCase + id::Int + name::String + pageversion::Symbol + node::N5Node + rows::Vector{Any} + schema::Vector{MD.SchemaElement} + paths::Vector{Vector{String}} + streams::Vector{N5LeafStream} + depth::Int + width::Int + astnodes::Int + leaves::Int + levelentries::Int + densevalues::Int + payloadbytes::Int +end + +mutable struct N5PropertyBuildState + rng::N5SplitMix64 + caseid::Int + serial::Int + maxwidth::Int + listordinal::Int + mapordinal::Int +end + +mutable struct N5PropertyValueState + rng::N5SplitMix64 +end + +mutable struct N5PropertyCoverage + nodes::Dict{Symbol,Int} + recursive_nodes::Dict{Symbol,Int} + list_layouts::Dict{Symbol,Int} + list_annotations::Dict{Symbol,Int} + map_markers::Dict{Symbol,Int} + entry_markers::Dict{Symbol,Int} + repetitions::Dict{Symbol,Int} + value_modes::Dict{Symbol,Int} + name_modes::Dict{Symbol,Int} + container_states::Dict{Symbol,Int} + element_states::Dict{Symbol,Int} + value_states::Dict{Symbol,Int} + collection_lengths::Dict{Int,Int} + depths::Dict{Int,Int} + widths::Dict{Int,Int} + row_counts::Dict{Int,Int} + page_versions::Dict{Symbol,Int} + physical_types::Dict{MD.Type.T,Int} + duplicate_maps::Int + complex_keys::Int + complex_values::Int + row_starts::Int + row_continuations::Int + rejected_candidates::Int +end + +function N5PropertyCoverage() + return N5PropertyCoverage( + Dict{Symbol,Int}(), Dict{Symbol,Int}(), Dict{Symbol,Int}(), + Dict{Symbol,Int}(), Dict{Symbol,Int}(), Dict{Symbol,Int}(), + Dict{Symbol,Int}(), Dict{Symbol,Int}(), Dict{Symbol,Int}(), + Dict{Symbol,Int}(), Dict{Symbol,Int}(), Dict{Symbol,Int}(), + Dict{Int,Int}(), Dict{Int,Int}(), Dict{Int,Int}(), Dict{Int,Int}(), + Dict{Symbol,Int}(), Dict{MD.Type.T,Int}(), 0, 0, 0, 0, 0, 0) +end + +function _n5splitmixvalue(value::UInt64) + mixed = value + mixed = (mixed ⊻ (mixed >> 30)) * UInt64(0xbf58476d1ce4e5b9) + mixed = (mixed ⊻ (mixed >> 27)) * UInt64(0x94d049bb133111eb) + return mixed ⊻ (mixed >> 31) +end + +function n5splitmix64!(rng::N5SplitMix64) + rng.state += UInt64(0x9e3779b97f4a7c15) + return _n5splitmixvalue(rng.state) +end + +function n5propertyrng(id::Integer, lane::UInt64=UInt64(0)) + 0 <= id <= N5_PROPERTY_CANDIDATE_LAST || throw(ArgumentError( + "N5 property candidate ID is outside 0:4095")) + identity = UInt64(id) * UInt64(0xd1342543de82ef95) + state = _n5splitmixvalue(N5_PROPERTY_SEED ⊻ identity ⊻ lane) + return N5SplitMix64(state) +end + +function _n5propertychoice!(rng::N5SplitMix64, count::Int) + count > 0 || throw(ArgumentError("N5 property choice is empty")) + return Int(rem(n5splitmix64!(rng), UInt64(count))) + 1 +end + +function _n5propertycoin!(rng::N5SplitMix64) + return isodd(n5splitmix64!(rng)) +end + +function _n5propertyname!(state::N5PropertyBuildState, prefix::String) + state.serial += 1 + return string(prefix, "_", state.caseid, "_", state.serial) +end + +function _n5propertyprimitive(state::N5PropertyBuildState, name::String, + optional::Bool) + choice = _n5propertychoice!(state.rng, 6) + if choice == 1 + return n5primitive(name, MD.Type.INT32; optional=optional) + elseif choice == 2 + return n5primitive(name, MD.Type.INT64; optional=optional) + elseif choice == 3 + return n5primitive(name, MD.Type.FLOAT; optional=optional) + elseif choice == 4 + return n5primitive(name, MD.Type.DOUBLE; optional=optional) + elseif choice == 5 + return n5primitive(name, MD.Type.BOOLEAN; optional=optional) + end + logical = _n5propertycoin!(state.rng) ? :string : :none + return n5primitive(name, MD.Type.BYTE_ARRAY; optional=optional, + logical=logical) +end + +function _n5propertyoptional(state::N5PropertyBuildState, + optional::Union{Nothing,Bool}) + optional === nothing || return optional + return _n5propertycoin!(state.rng) +end + +function _n5propertystruct(state::N5PropertyBuildState, name::String, + depth::Int, optional::Bool) + childcount = state.maxwidth + fields = N5Node[] + sizehint!(fields, childcount) + for index in 1:childcount + childname = _n5propertyname!(state, "f") + if index == 1 && depth > 1 + push!(fields, _n5propertynode(state, childname, depth - 1)) + else + push!(fields, _n5propertyprimitive(state, childname, + _n5propertycoin!(state.rng))) + end + end + return n5struct(name, fields; optional=optional) +end + +function _n5propertylist(state::N5PropertyBuildState, name::String, + depth::Int, layout::Symbol, optional::Bool; rule3map::Bool=false, + tuplewrapper::Bool=false) + state.listordinal += 1 + annotation = N5_PROPERTY_LIST_ANNOTATIONS[mod(state.caseid + + state.listordinal - 1, length(N5_PROPERTY_LIST_ANNOTATIONS)) + 1] + if layout === :rule1 + element = _n5propertyprimitive(state, + _n5propertyname!(state, "element"), false) + return n5list(name, element; optional=optional, layout=:rule1, + annotation=annotation) + elseif layout === :rule2 + childcount = max(2, state.maxwidth) + fields = N5Node[] + sizehint!(fields, childcount) + for index in 1:childcount + childname = _n5propertyname!(state, "field") + if index == 1 && depth > 2 + push!(fields, _n5propertynode(state, childname, depth - 2)) + else + push!(fields, _n5propertyprimitive(state, childname, + _n5propertycoin!(state.rng))) + end + end + element = n5struct(_n5propertyname!(state, "element"), fields) + return n5list(name, element; optional=optional, layout=:rule2, + wrapper="array", annotation=annotation) + elseif layout === :rule3 + innername = _n5propertyname!(state, rule3map ? "map" : "array") + if rule3map + element = _n5propertymap(state, innername, max(depth - 1, 2), false) + else + element = _n5propertylist(state, innername, max(depth - 1, 2), + :canonical, false) + end + return n5list(name, element; optional=optional, layout=:rule3, + wrapper=rule3map ? "map" : "array", annotation=annotation) + elseif layout === :rule4 + child = depth > 2 ? _n5propertynode(state, + _n5propertyname!(state, "value"), depth - 2) : + _n5propertyprimitive(state, _n5propertyname!(state, "value"), + _n5propertycoin!(state.rng)) + wrapper = tuplewrapper ? string(name, "_tuple") : "array" + element = n5struct(wrapper, N5Node[child]) + return n5list(name, element; optional=optional, layout=:rule4, + wrapper=wrapper, annotation=annotation) + end + child = depth > 1 ? _n5propertynode(state, + _n5propertyname!(state, "element"), depth - 1) : + _n5propertyprimitive(state, _n5propertyname!(state, "element"), false) + if layout === :rule5 + wrappers = ("Array", "ARRAY", string(name, "_Tuple"), "list") + wrapper = wrappers[mod(state.caseid + state.listordinal - 1, + length(wrappers)) + 1] + return n5list(name, child; optional=optional, layout=:rule5, + wrapper=wrapper, annotation=annotation) + end + return n5list(name, child; optional=optional, layout=:canonical, + wrapper="list", annotation=annotation) +end + +function _n5propertymap(state::N5PropertyBuildState, name::String, + depth::Int, optional::Bool) + state.mapordinal += 1 + ordinal = state.caseid + state.mapordinal - 1 + marker = N5_PROPERTY_MAP_MARKERS[mod(ordinal, + length(N5_PROPERTY_MAP_MARKERS)) + 1] + entrymarker = N5_PROPERTY_ENTRY_MARKERS[mod(div(ordinal, + length(N5_PROPERTY_MAP_MARKERS)), + length(N5_PROPERTY_ENTRY_MARKERS)) + 1] + valuemode = (:absent, :required, :optional)[mod(div(ordinal, 3), 3) + 1] + arbitrary = isodd(div(ordinal, 5)) + entryname = arbitrary ? _n5propertyname!(state, "entries") : "key_value" + keyname = arbitrary ? _n5propertyname!(state, "left") : "key" + valuename = arbitrary ? _n5propertyname!(state, "right") : "value" + recursivekey = depth > 2 && (valuemode === :absent || isodd(ordinal)) + if recursivekey + key = _n5propertynode(state, keyname, depth - 1; optional=false) + else + optionalkey = isodd(div(ordinal, 7)) + key = _n5propertyprimitive(state, keyname, optionalkey) + end + value = nothing + if valuemode !== :absent + valueoptional = valuemode === :optional + if depth > 2 && !recursivekey + value = _n5propertynode(state, valuename, depth - 1; + optional=valueoptional) + else + value = _n5propertyprimitive(state, valuename, valueoptional) + end + end + return n5map(name, key, value; optional=optional, marker=marker, + entrymarker=entrymarker, entryname=entryname) +end + +function _n5propertynode(state::N5PropertyBuildState, name::String, + depth::Int; optional::Union{Nothing,Bool}=nothing) + depth >= 1 || throw(ArgumentError("N5 property depth is below one")) + optionalvalue = _n5propertyoptional(state, optional) + depth == 1 && return _n5propertyprimitive(state, name, optionalvalue) + kind = _n5propertychoice!(state.rng, 3) + kind == 1 && return _n5propertystruct(state, name, depth, optionalvalue) + if kind == 2 + layouts = depth >= 3 ? N5_PROPERTY_LIST_LAYOUTS : + (:canonical, :rule1, :rule5) + layout = layouts[_n5propertychoice!(state.rng, length(layouts))] + return _n5propertylist(state, name, depth, layout, optionalvalue; + rule3map=_n5propertycoin!(state.rng), + tuplewrapper=_n5propertycoin!(state.rng)) + end + return _n5propertymap(state, name, depth, optionalvalue) +end + +function _n5propertyroot(state::N5PropertyBuildState) + mode = mod(state.caseid, 16) + targetdepth = mod(state.caseid, 6) + 1 + optional = isodd(div(state.caseid, 2)) + name = "field" + mode == 0 && return _n5propertyprimitive(state, name, optional) + mode in (1, 12, 15) && return _n5propertystruct(state, name, + max(targetdepth, 2), optional) + mode == 2 && return _n5propertylist(state, name, max(targetdepth, 2), + :canonical, optional) + mode == 3 && return _n5propertylist(state, name, 2, :rule1, optional) + mode == 4 && return _n5propertylist(state, name, max(targetdepth, 3), + :rule2, optional) + mode == 5 && return _n5propertylist(state, name, max(targetdepth, 3), + :rule3, optional) + mode == 6 && return _n5propertylist(state, name, max(targetdepth, 3), + :rule3, optional; rule3map=true) + mode == 7 && return _n5propertylist(state, name, max(targetdepth, 3), + :rule4, optional) + mode == 8 && return _n5propertylist(state, name, max(targetdepth, 3), + :rule4, optional; tuplewrapper=true) + mode == 9 && return _n5propertylist(state, name, max(targetdepth, 2), + :rule5, optional) + mode in (10, 11, 14) && return _n5propertymap(state, name, + max(targetdepth, 2), optional) + return _n5propertylist(state, name, max(targetdepth, 2), + N5_PROPERTY_LIST_LAYOUTS[_n5propertychoice!(state.rng, + length(N5_PROPERTY_LIST_LAYOUTS))], optional; + rule3map=_n5propertycoin!(state.rng), + tuplewrapper=_n5propertycoin!(state.rng)) +end + +function _n5propertyphase(index::Int, salt::Int, count::Int) + return mod(index - 1 + salt, count) +end + +function _n5propertyprimitivevalue(state::N5PropertyValueState, + node::N5Primitive) + raw = n5splitmix64!(state.rng) + if node.physical == MD.Type.INT32 + return Int32(Int(rem(raw, UInt64(2001))) - 1000) + elseif node.physical == MD.Type.INT64 + return Int64(raw & UInt64(0x0000ffffffffffff)) - Int64(1 << 46) + elseif node.physical == MD.Type.FLOAT + return Float32(Int(rem(raw, UInt64(2001))) - 1000) / Float32(7) + elseif node.physical == MD.Type.DOUBLE + return Float64(Int(rem(raw, UInt64(2001))) - 1000) / 11.0 + elseif node.physical == MD.Type.BOOLEAN + return isodd(raw) + elseif node.physical == MD.Type.BYTE_ARRAY + lengthvalue = Int(rem(raw >> 8, UInt64(8))) + 1 + bytes = UInt8[UInt8((raw >> (8 * mod(index, 8))) & 0x7f) + for index in 0:(lengthvalue - 1)] + if node.logical === :string + for index in eachindex(bytes) + bytes[index] = UInt8('a') + bytes[index] % UInt8(26) + end + return String(bytes) + end + return bytes + end + throw(ArgumentError("unsupported N5 property primitive $(node.physical)")) +end + +function _n5propertybatch(state::N5PropertyValueState, node::N5Primitive, + count::Int, salt::Int; forcepresent::Bool=false) + values = Vector{Any}(undef, count) + for index in 1:count + absent = node.optional && !forcepresent && + iszero(_n5propertyphase(index, salt, 2)) + values[index] = absent ? missing : _n5propertyprimitivevalue(state, node) + end + return values +end + +function _n5propertybatch(state::N5PropertyValueState, node::N5Struct, + count::Int, salt::Int; forcepresent::Bool=false) + present = Int[] + for index in 1:count + absent = node.optional && !forcepresent && + iszero(_n5propertyphase(index, salt, 2)) + absent || push!(present, index) + end + children = Vector{Vector{Any}}(undef, length(node.fields)) + for (index, field) in enumerate(node.fields) + children[index] = _n5propertybatch(state, field, length(present), + salt + 17 * index) + end + values = Vector{Any}(undef, count) + fill!(values, missing) + for (slot, index) in enumerate(present) + values[index] = N5Record(Any[child[slot] for child in children]) + end + return values +end + +function _n5propertypresentlength(state::N5PropertyValueState, + optionalchild::Bool) + lengthvalue = _n5propertychoice!(state.rng, + N5_PROPERTY_MAX_COLLECTION_LENGTH) + optionalchild && (lengthvalue = max(lengthvalue, 3)) + return lengthvalue +end + +function _n5propertybatch(state::N5PropertyValueState, node::N5List, + count::Int, salt::Int; forcepresent::Bool=false) + lengths = zeros(Int, count) + missingrows = falses(count) + for index in 1:count + if node.optional && !forcepresent + phase = _n5propertyphase(index, salt, 3) + missingrows[index] = phase == 0 + phase == 2 && (lengths[index] = _n5propertypresentlength(state, + node.element.optional)) + else + phase = _n5propertyphase(index, salt, 2) + phase == 1 && (lengths[index] = _n5propertypresentlength(state, + node.element.optional)) + end + end + total = sum(lengths) + elements = _n5propertybatch(state, node.element, total, salt + 31) + values = Vector{Any}(undef, count) + cursor = 1 + for index in 1:count + if missingrows[index] + values[index] = missing + else + lengthvalue = lengths[index] + values[index] = lengthvalue == 0 ? [] : + Any[elements[cursor:(cursor + lengthvalue - 1)]...] + cursor += lengthvalue + end + end + return values +end + +function _n5propertymaplength(state::N5PropertyValueState, node::N5Map) + optionalchild = node.value !== nothing && node.value.optional + lengthvalue = _n5propertypresentlength(state, optionalchild) + return max(lengthvalue, 2) +end + +function _n5propertybatch(state::N5PropertyValueState, node::N5Map, + count::Int, salt::Int; forcepresent::Bool=false) + lengths = zeros(Int, count) + missingrows = falses(count) + for index in 1:count + if node.optional && !forcepresent + phase = _n5propertyphase(index, salt, 3) + missingrows[index] = phase == 0 + phase == 2 && (lengths[index] = _n5propertymaplength(state, node)) + else + phase = _n5propertyphase(index, salt, 2) + phase == 1 && (lengths[index] = _n5propertymaplength(state, node)) + end + end + total = sum(lengths) + keys = _n5propertybatch(state, node.key, total, salt + 43; + forcepresent=true) + mapvalues = node.value === nothing ? nothing : + _n5propertybatch(state, node.value, total, salt + 59) + cursor = 1 + for lengthvalue in lengths + if lengthvalue >= 2 + keys[cursor + 1] = keys[cursor] + end + cursor += lengthvalue + end + values = Vector{Any}(undef, count) + cursor = 1 + for index in 1:count + if missingrows[index] + values[index] = missing + continue + end + entries = N5Entry[] + sizehint!(entries, lengths[index]) + for offset in 0:(lengths[index] - 1) + position = cursor + offset + if mapvalues === nothing + push!(entries, N5Entry(keys[position], nothing, false)) + else + push!(entries, N5Entry(keys[position], mapvalues[position], true)) + end + end + values[index] = N5MapValue(entries) + cursor += lengths[index] + end + return values +end + +function _n5propertyempty(value) + value isa AbstractVector && return isempty(value) + value isa N5MapValue && return isempty(value.entries) + return false +end + +function _n5propertystatesvalid(node::N5Primitive, values::Vector{Any}; + forcepresent::Bool=false) + if node.optional && !forcepresent && length(values) >= 2 + any(ismissing, values) && any(!ismissing, values) || return false + end + return forcepresent ? all(!ismissing, values) : true +end + +function _n5propertystatesvalid(node::N5Struct, values::Vector{Any}; + forcepresent::Bool=false) + if node.optional && !forcepresent && length(values) >= 2 + any(ismissing, values) && any(!ismissing, values) || return false + end + forcepresent && !all(!ismissing, values) && return false + present = Any[value for value in values if !ismissing(value)] + for (index, child) in enumerate(node.fields) + childvalues = Any[value.values[index] for value in present] + _n5propertystatesvalid(child, childvalues) || return false + end + return true +end + +function _n5propertycontainerstates(node, values::Vector{Any}, + forcepresent::Bool) + if node.optional && !forcepresent && length(values) >= 3 + any(ismissing, values) || return false + any(value -> !ismissing(value) && _n5propertyempty(value), values) || + return false + any(value -> !ismissing(value) && !_n5propertyempty(value), values) || + return false + elseif (!node.optional || forcepresent) && length(values) >= 2 + all(!ismissing, values) || return false + any(_n5propertyempty, values) || return false + any(value -> !_n5propertyempty(value), values) || return false + end + return true +end + +function _n5propertystatesvalid(node::N5List, values::Vector{Any}; + forcepresent::Bool=false) + _n5propertycontainerstates(node, values, forcepresent) || return false + elements = [] + for value in values + ismissing(value) || append!(elements, value) + end + return _n5propertystatesvalid(node.element, elements) +end + +function _n5propertystatesvalid(node::N5Map, values::Vector{Any}; + forcepresent::Bool=false) + _n5propertycontainerstates(node, values, forcepresent) || return false + keys = [] + mapvalues = [] + for value in values + ismissing(value) && continue + if !isempty(value.entries) + length(value.entries) >= 2 || return false + isequal(value.entries[1].key, value.entries[2].key) || return false + end + for entry in value.entries + ismissing(entry.key) && return false + push!(keys, entry.key) + node.value === nothing || push!(mapvalues, entry.value) + end + end + _n5propertystatesvalid(node.key, keys; forcepresent=true) || return false + node.value === nothing && return true + return _n5propertystatesvalid(node.value, mapvalues) +end + +function n5propertyastnodes(node::N5Primitive) + return 1 +end + +function n5propertyastnodes(node::N5Struct) + return 1 + sum(n5propertyastnodes, node.fields; init=0) +end + +function n5propertyastnodes(node::N5List) + return 1 + n5propertyastnodes(node.element) +end + +function n5propertyastnodes(node::N5Map) + value = node.value === nothing ? 0 : n5propertyastnodes(node.value) + return 1 + n5propertyastnodes(node.key) + value +end + +function n5propertydepth(node::N5Primitive) + return 1 +end + +function n5propertydepth(node::N5Struct) + return 1 + maximum(n5propertydepth, node.fields) +end + +function n5propertydepth(node::N5List) + return 1 + n5propertydepth(node.element) +end + +function n5propertydepth(node::N5Map) + value = node.value === nothing ? 0 : n5propertydepth(node.value) + return 1 + max(n5propertydepth(node.key), value) +end + +function n5propertywidth(node::N5Primitive) + return 1 +end + +function n5propertywidth(node::N5Struct) + return max(length(node.fields), maximum(n5propertywidth, node.fields)) +end + +function n5propertywidth(node::N5List) + return max(1, n5propertywidth(node.element)) +end + +function n5propertywidth(node::N5Map) + own = node.value === nothing ? 1 : 2 + value = node.value === nothing ? 1 : n5propertywidth(node.value) + return max(own, n5propertywidth(node.key), value) +end + +function _n5propertyunarychain(name::String, depth::Int) + node = n5primitive(string(name, "_leaf"), MD.Type.INT32) + for level in 2:depth + node = n5struct(string(name, "_level_", level), N5Node[node]) + end + return node +end + +function _n5propertyastcapnode() + groups = N5Node[] + for groupindex in 1:4 + fields = N5Node[] + for fieldindex in 1:4 + name = string("cap_", groupindex, "_", fieldindex) + push!(fields, _n5propertyunarychain(name, 4)) + end + push!(groups, n5struct(string("cap_group_", groupindex), fields)) + end + return n5struct("field", groups) +end + +function _n5propertyleafcapnode() + groups = N5Node[] + for groupindex in 1:4 + fields = N5Node[n5primitive( + string("cap_leaf_", groupindex, "_", fieldindex), MD.Type.INT32) + for fieldindex in 1:4] + push!(groups, n5struct(string("cap_group_", groupindex), fields)) + end + branch = n5struct("cap_branch", groups) + tail = n5primitive("cap_tail", MD.Type.INT32) + return n5struct("field", N5Node[branch, tail]) +end + +function _n5propertynestedmissing(depth::Int) + value = missing + for _ in 1:depth + value = Any[value, value, value, value, value] + end + return value +end + +function _n5propertynestedvalue(depth::Int, value) + iszero(depth) && return value + missingbranch = _n5propertynestedmissing(depth - 1) + return Any[_n5propertynestedvalue(depth - 1, value), missingbranch, + missingbranch, missingbranch, missingbranch] +end + +function _n5propertyhardcapcandidate(id::Int) + if id == 29 + node = _n5propertyunarychain("field", 7) + values = N5PropertyValueState(n5propertyrng(id, + UInt64(0x56414c5545534e35))) + return node, _n5propertybatch(values, node, 1, id) + elseif id == 63 + fields = N5Node[n5primitive(string("wide_", index), MD.Type.INT32) + for index in 1:5] + node = n5struct("field", fields) + values = N5PropertyValueState(n5propertyrng(id, + UInt64(0x56414c5545534e35))) + return node, _n5propertybatch(values, node, 1, id) + elseif id == 95 + node = _n5propertyastcapnode() + values = N5PropertyValueState(n5propertyrng(id, + UInt64(0x56414c5545534e35))) + return node, _n5propertybatch(values, node, 1, id) + elseif id == 154 + node = _n5propertyleafcapnode() + values = N5PropertyValueState(n5propertyrng(id, + UInt64(0x56414c5545534e35))) + return node, _n5propertybatch(values, node, 1, id) + elseif id == 201 + node = n5primitive("entry", MD.Type.INT32; optional=true) + for level in 1:5 + node = n5list(string("level_", level), node) + end + nested = _n5propertynestedmissing(5) + values = Any[nested for _ in 1:24] + rng = n5propertyrng(id, UInt64(0x56414c5545534e35)) + present = Int32(rem(n5splitmix64!(rng), UInt64(2001))) - Int32(1000) + values[1] = _n5propertynestedvalue(5, present) + return node, values + elseif id == 254 + node = n5primitive("field", MD.Type.BYTE_ARRAY) + rng = n5propertyrng(id, UInt64(0x56414c5545534e35)) + byte = UInt8(n5splitmix64!(rng) & UInt64(0xff)) + bytes = fill(byte, N5_PROPERTY_MAX_PAYLOAD_BYTES + 1) + return node, Any[bytes] + end + return nothing +end + +function _n5propertygeneratedcandidate(id::Int) + hardcap = _n5propertyhardcapcandidate(id) + hardcap !== nothing && return hardcap + build = N5PropertyBuildState(n5propertyrng(id), id, 0, + mod(div(id, 6), 4) + 1, 0, 0) + node = _n5propertyroot(build) + values = N5PropertyValueState(n5propertyrng(id, + UInt64(0x56414c5545534e35))) + return node, _n5propertybatch(values, node, mod(id, 25), id) +end + +function n5propertycandidateassessment(id::Int) + 0 <= id <= N5_PROPERTY_CANDIDATE_LAST || throw(ArgumentError( + "N5 property candidate ID is outside 0:4095")) + node, rows = _n5propertygeneratedcandidate(id) + statesvalid = _n5propertystatesvalid(node, rows) + compiled = n5compile(node) + streams = n5shred(compiled, rows) + schema = n5schema(node) + physical = n5physicalleaves(schema) + astnodes = n5propertyastnodes(node) + depth = n5propertydepth(node) + width = n5propertywidth(node) + leaves = length(compiled.leaves) + levelentries = sum(stream -> length(stream.repetition), streams; init=0) + densevalues = sum(stream -> length(stream.values), streams; init=0) + payloadbytes = sum(length(_n5plainencode(stream.values, leaf)) + for (stream, leaf) in zip(streams, physical); init=0) + reason = depth < 1 || depth > 6 ? :depth : + width < 1 || width > 4 ? :width : + astnodes > N5_PROPERTY_MAX_AST_NODES ? :astnodes : + leaves > N5_PROPERTY_MAX_LEAVES ? :leaves : + levelentries > N5_PROPERTY_MAX_LEVEL_ENTRIES ? :levelentries : + densevalues > N5_PROPERTY_MAX_DENSE_VALUES ? :densevalues : + payloadbytes > N5_PROPERTY_MAX_PAYLOAD_BYTES ? :payloadbytes : + !statesvalid ? :semantic_states : nothing + paths = Vector{String}[copy(leaf.path) for leaf in physical] + metrics = (; depth, width, astnodes, leaves, levelentries, densevalues, + payloadbytes) + reason === nothing || return (; candidate=nothing, reason, metrics) + candidate = (; id, node, rows, schema, paths, streams, depth, width, astnodes, + leaves, levelentries, densevalues, payloadbytes) + return (; candidate, reason=nothing, metrics) +end + +function _n5propertycandidate(id::Int) + return n5propertycandidateassessment(id).candidate +end + +function n5propertycases() + cases = N5PropertyCase[] + rejected = 0 + for id in 0:N5_PROPERTY_CANDIDATE_LAST + candidate = _n5propertycandidate(id) + if candidate === nothing + rejected += 1 + continue + end + pageversion = isodd(length(cases)) ? :v2 : :v1 + name = string("generated-", lpad(string(id), 4, '0')) + push!(cases, N5PropertyCase(candidate.id, name, pageversion, + candidate.node, candidate.rows, candidate.schema, candidate.paths, + candidate.streams, candidate.depth, candidate.width, + candidate.astnodes, candidate.leaves, candidate.levelentries, + candidate.densevalues, candidate.payloadbytes)) + length(cases) == N5_PROPERTY_CASE_COUNT && break + end + length(cases) == N5_PROPERTY_CASE_COUNT || throw(ArgumentError( + "N5 property schedule accepted $(length(cases)) of 256 cases")) + return cases, rejected +end + +function _n5propertybump!(counts::Dict{T,Int}, key::T) where {T} + counts[key] = get(() -> 0, counts, key) + 1 + return +end + +function _n5propertycoverast!(coverage::N5PropertyCoverage, + node::N5Primitive, level::Int) + _n5propertybump!(coverage.nodes, :primitive) + level > 1 && _n5propertybump!(coverage.recursive_nodes, :primitive) + _n5propertybump!(coverage.repetitions, + node.optional ? :optional : :required) + _n5propertybump!(coverage.physical_types, node.physical) + return +end + +function _n5propertycoverast!(coverage::N5PropertyCoverage, + node::N5Struct, level::Int) + _n5propertybump!(coverage.nodes, :struct) + level > 1 && _n5propertybump!(coverage.recursive_nodes, :struct) + _n5propertybump!(coverage.repetitions, + node.optional ? :optional : :required) + for child in node.fields + _n5propertycoverast!(coverage, child, level + 1) + end + return +end + +function _n5propertycoverast!(coverage::N5PropertyCoverage, + node::N5List, level::Int) + _n5propertybump!(coverage.nodes, :list) + level > 1 && _n5propertybump!(coverage.recursive_nodes, :list) + _n5propertybump!(coverage.repetitions, + node.optional ? :optional : :required) + _n5propertybump!(coverage.repetitions, :repeated) + _n5propertybump!(coverage.list_layouts, node.layout) + _n5propertybump!(coverage.list_annotations, node.annotation) + _n5propertycoverast!(coverage, node.element, level + 1) + return +end + +function _n5propertycoverast!(coverage::N5PropertyCoverage, + node::N5Map, level::Int) + _n5propertybump!(coverage.nodes, :map) + level > 1 && _n5propertybump!(coverage.recursive_nodes, :map) + _n5propertybump!(coverage.repetitions, + node.optional ? :optional : :required) + _n5propertybump!(coverage.repetitions, :repeated) + _n5propertybump!(coverage.map_markers, node.marker) + _n5propertybump!(coverage.entry_markers, node.entrymarker) + valuemode = node.value === nothing ? :absent : + node.value.optional ? :optional : :required + _n5propertybump!(coverage.value_modes, valuemode) + names = node.entryname == "key_value" && node.key.name == "key" && + (node.value === nothing || node.value.name == "value") ? + :canonical : :arbitrary + _n5propertybump!(coverage.name_modes, names) + node.key isa N5Primitive || (coverage.complex_keys += 1) + node.value === nothing || node.value isa N5Primitive || + (coverage.complex_values += 1) + _n5propertycoverast!(coverage, node.key, level + 1) + node.value === nothing || _n5propertycoverast!(coverage, + node.value, level + 1) + return +end + +function _n5propertycoverstate!(coverage::N5PropertyCoverage, value) + if ismissing(value) + _n5propertybump!(coverage.container_states, :null) + elseif _n5propertyempty(value) + _n5propertybump!(coverage.container_states, :empty) + else + _n5propertybump!(coverage.container_states, :present) + end + return +end + +function _n5propertycovervalues!(coverage::N5PropertyCoverage, + node::N5Primitive, values::Vector{Any}) + return +end + +function _n5propertycovervalues!(coverage::N5PropertyCoverage, + node::N5Struct, values::Vector{Any}) + present = Any[value for value in values if !ismissing(value)] + for (index, child) in enumerate(node.fields) + childvalues = Any[value.values[index] for value in present] + _n5propertycovervalues!(coverage, child, childvalues) + end + return +end + +function _n5propertycovervalues!(coverage::N5PropertyCoverage, + node::N5List, values::Vector{Any}) + elements = [] + for value in values + _n5propertycoverstate!(coverage, value) + if !ismissing(value) + _n5propertybump!(coverage.collection_lengths, length(value)) + append!(elements, value) + end + end + if node.element.optional + for value in elements + _n5propertybump!(coverage.element_states, + ismissing(value) ? :null : :present) + end + end + _n5propertycovervalues!(coverage, node.element, elements) + return +end + +function _n5propertycovervalues!(coverage::N5PropertyCoverage, + node::N5Map, values::Vector{Any}) + keys = [] + mapvalues = [] + for value in values + _n5propertycoverstate!(coverage, value) + ismissing(value) && continue + _n5propertybump!(coverage.collection_lengths, length(value.entries)) + if length(value.entries) >= 2 && + isequal(value.entries[1].key, value.entries[2].key) + coverage.duplicate_maps += 1 + end + for entry in value.entries + push!(keys, entry.key) + node.value === nothing || push!(mapvalues, entry.value) + end + end + _n5propertycovervalues!(coverage, node.key, keys) + if node.value !== nothing + if node.value.optional + for value in mapvalues + _n5propertybump!(coverage.value_states, + ismissing(value) ? :null : :present) + end + end + _n5propertycovervalues!(coverage, node.value, mapvalues) + end + return +end + +function n5propertycoverage(cases::Vector{N5PropertyCase}, rejected::Int) + coverage = N5PropertyCoverage() + coverage.rejected_candidates = rejected + for case in cases + _n5propertybump!(coverage.depths, case.depth) + _n5propertybump!(coverage.widths, case.width) + _n5propertybump!(coverage.row_counts, length(case.rows)) + _n5propertybump!(coverage.page_versions, case.pageversion) + _n5propertycoverast!(coverage, case.node, 1) + _n5propertycovervalues!(coverage, case.node, case.rows) + for stream in case.streams + coverage.row_starts += count(iszero, stream.repetition) + coverage.row_continuations += count(!iszero, stream.repetition) + end + end + return coverage +end + +function _n5propertymissing!(missing::Vector{String}, counts, key, label::String) + get(() -> 0, counts, key) > 0 || push!(missing, label) + return +end + +function n5propertymissingcoverage(coverage::N5PropertyCoverage) + missing = String[] + for kind in (:primitive, :struct, :list, :map) + _n5propertymissing!(missing, coverage.nodes, kind, "node:$kind") + end + for kind in (:primitive, :struct, :list, :map) + _n5propertymissing!(missing, coverage.recursive_nodes, kind, + "recursive-node:$kind") + end + for layout in N5_PROPERTY_LIST_LAYOUTS + _n5propertymissing!(missing, coverage.list_layouts, layout, + "list-layout:$layout") + end + for annotation in N5_PROPERTY_LIST_ANNOTATIONS + _n5propertymissing!(missing, coverage.list_annotations, annotation, + "list-annotation:$annotation") + end + for marker in N5_PROPERTY_MAP_MARKERS + _n5propertymissing!(missing, coverage.map_markers, marker, + "map-marker:$marker") + end + for marker in N5_PROPERTY_ENTRY_MARKERS + _n5propertymissing!(missing, coverage.entry_markers, marker, + "entry-marker:$marker") + end + for repetition in (:required, :optional, :repeated) + _n5propertymissing!(missing, coverage.repetitions, repetition, + "repetition:$repetition") + end + for mode in (:absent, :required, :optional) + _n5propertymissing!(missing, coverage.value_modes, mode, + "map-value:$mode") + end + for mode in (:canonical, :arbitrary) + _n5propertymissing!(missing, coverage.name_modes, mode, + "map-names:$mode") + end + for state in (:null, :empty, :present) + _n5propertymissing!(missing, coverage.container_states, state, + "container:$state") + end + for state in (:null, :present) + _n5propertymissing!(missing, coverage.element_states, state, + "optional-element:$state") + _n5propertymissing!(missing, coverage.value_states, state, + "optional-value:$state") + end + for lengthvalue in 0:N5_PROPERTY_MAX_COLLECTION_LENGTH + _n5propertymissing!(missing, coverage.collection_lengths, lengthvalue, + "collection-length:$lengthvalue") + end + for depth in 1:6 + _n5propertymissing!(missing, coverage.depths, depth, "depth:$depth") + end + for width in 1:4 + _n5propertymissing!(missing, coverage.widths, width, "width:$width") + end + for rows in 0:24 + _n5propertymissing!(missing, coverage.row_counts, rows, "rows:$rows") + end + for version in (:v1, :v2) + _n5propertymissing!(missing, coverage.page_versions, version, + "page:$version") + end + for physical in (MD.Type.INT32, MD.Type.INT64, MD.Type.FLOAT, + MD.Type.DOUBLE, MD.Type.BOOLEAN, MD.Type.BYTE_ARRAY) + _n5propertymissing!(missing, coverage.physical_types, physical, + "physical:$(physical.value)") + end + coverage.duplicate_maps > 0 || push!(missing, "duplicate-map-key") + coverage.complex_keys > 0 || push!(missing, "recursive-map-key") + coverage.complex_values > 0 || push!(missing, "recursive-map-value") + coverage.row_starts > 0 || push!(missing, "row-start") + coverage.row_continuations > 0 || push!(missing, "row-continuation") + coverage.rejected_candidates > 0 || push!(missing, "rejected-candidate") + return missing +end + +function n5propertycodecsubset(cases::Vector{N5PropertyCase}) + byid = Dict(case.id => case for case in cases) + subset = N5PropertyCase[] + sizehint!(subset, N5_PROPERTY_CODEC_COUNT) + for id in N5_PROPERTY_CODEC_CASE_IDS + haskey(byid, id) || throw(ArgumentError( + "N5 property codec case ID $id is not accepted")) + case = byid[id] + length(case.rows) > 0 && case.payloadbytes > 0 || throw(ArgumentError( + "N5 property codec case ID $id has no writable payload")) + push!(subset, case) + end + length(subset) == N5_PROPERTY_CODEC_COUNT || throw(ArgumentError( + "N5 property codec subset does not contain 32 cases")) + return subset +end diff --git a/test/conformance/n5/model/runtests.jl b/test/conformance/n5/model/runtests.jl new file mode 100644 index 0000000..6ea958f --- /dev/null +++ b/test/conformance/n5/model/runtests.jl @@ -0,0 +1,906 @@ +using Parquet +using SHA +using Test + +include(joinpath(@__DIR__, "N5ConformanceModel.jl")) + +const N5 = N5ConformanceModel +const N5_PROPERTY_CASES, N5_PROPERTY_REJECTED = N5.n5propertycases() + +function n5frozenwriterreference(expression) + expression isa Expr && expression.head == :. || return nothing + length(expression.args) == 2 || return nothing + expression.args[1] == :Parquet || return nothing + name = expression.args[2] + name isa QuoteNode || return nothing + name.value in (:_encodefile, :write) || return nothing + return name.value +end + +function n5frozenwriterusesoptions(expression::Expr) + for argument in expression.args + argument isa Expr && argument.head == :parameters || continue + for keyword in argument.args + keyword isa Expr && keyword.head == :... || continue + length(keyword.args) == 1 && keyword.args[1] == :options && + return true + end + end + return false +end + +function n5collectfrozenwritercalls!(calls::Vector{Tuple{Symbol,Bool}}, + expression) + expression isa Expr || return calls + if expression.head == :call && !isempty(expression.args) + name = n5frozenwriterreference(expression.args[1]) + if name !== nothing + push!(calls, (name, n5frozenwriterusesoptions(expression))) + for argument in expression.args[2:end] + n5collectfrozenwritercalls!(calls, argument) + end + return calls + end + end + name = n5frozenwriterreference(expression) + name === nothing || push!(calls, (name, false)) + for argument in expression.args + n5collectfrozenwritercalls!(calls, argument) + end + return calls +end + +function n5frozenwriterguard() + root = normpath(joinpath(@__DIR__, "..")) + integration = normpath(joinpath(@__DIR__, "integration.jl")) + violations = String[] + calls = Tuple{Symbol,Bool}[] + for (directory, _, files) in walkdir(root) + for file in sort!(files) + endswith(file, ".jl") || continue + path = normpath(joinpath(directory, file)) + relative = relpath(path, root) + first(splitpath(relative)) == "hardening" && continue + parsed = Meta.parseall(read(path, String)) + found = Tuple{Symbol,Bool}[] + n5collectfrozenwritercalls!(found, parsed) + if path == integration + append!(calls, found) + else + append!(violations, + ("$relative:$name" for (name, _) in found)) + end + end + end + append!(violations, + ("model/integration.jl:$name" for (name, usesoptions) in calls + if !usesoptions)) + return (; integration, calls, violations) +end + +@testset "N5 frozen outputs disable writer statistics" begin + guard = n5frozenwriterguard() + @test guard.violations == String[] + @test guard.calls == [(:_encodefile, true), (:_encodefile, true), + (:write, true), (:write, true)] + source = read(guard.integration, String) + helper = match(r"(?ms)^function n5productionencodedbytes\b.*?^end$", + source) + @test helper !== nothing + if helper !== nothing + options = match(r"(?s)options\s*=\s*\(;(.*?)\)", helper.match) + @test options !== nothing + if options !== nothing + @test occursin(r"\bstatistics\s*=\s*false\b", options.match) + end + helpercalls = Tuple{Symbol,Bool}[] + n5collectfrozenwritercalls!(helpercalls, Meta.parse(helper.match)) + @test helpercalls == guard.calls + end +end + +function n5checkcase(case) + compiled = N5.n5compile(case.node) + physical = N5.n5physicalleaves(case.schema) + @test length(compiled.leaves) == length(case.streams) + @test length(physical) == length(case.streams) + @test [leaf.path for leaf in physical] == case.paths + @test [leaf.max_repetition for leaf in physical] == + [stream.max_repetition for stream in case.streams] + @test [leaf.max_definition for leaf in physical] == + [stream.max_definition for stream in case.streams] + @test [leaf.max_repetition for leaf in compiled.leaves] == + [stream.max_repetition for stream in case.streams] + @test [leaf.max_definition for leaf in compiled.leaves] == + [stream.max_definition for stream in case.streams] + actual = N5.n5shred(compiled, case.rows) + @test isequal(actual, case.streams) + @test isequal(N5.n5assemble(compiled, actual, length(case.rows)), case.rows) + return compiled +end + +function n5checkbytes(case, compiled, pageversion::Symbol) + firstbytes = N5.n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=pageversion) + @test firstbytes == N5.n5emitfile(case.schema, case.streams, + length(case.rows); pageversion=pageversion) + decoded = N5.n5decodefile(firstbytes) + @test decoded.metadata.schema == case.schema + @test decoded.metadata.schema[1].name == "schema" + @test decoded.metadata.num_rows == length(case.rows) + @test [leaf.path for leaf in decoded.leaves] == case.paths + @test all(==(pageversion), decoded.pageversions) + @test isequal(decoded.streams, case.streams) + @test isequal(N5.n5assemble(compiled, decoded.streams, + length(case.rows)), case.rows) + return firstbytes +end + +function n5productioncheck(node, schema, streams, expectedrows, + pageversion::Symbol) + compiled = N5.n5compile(node) + bytes = N5.n5emitfile(schema, streams, length(expectedrows); + pageversion=pageversion) + table = Parquet.Table(bytes) + try + assembled = N5.n5assemble(compiled, streams, length(expectedrows)) + @test isequal(assembled, expectedrows) + @test isequal(N5.n5normalizetable(node, table), assembled) + expectedtree = N5.n5expectedvectortree(node, assembled) + @test N5.n5comparevectortree(expectedtree, table.columns) + actual, fields, rows = N5.n5productionstreams(table, compiled) + @test rows == length(expectedrows) + @test fields.elements == schema + @test N5.n5compareproductionstreams(actual, streams, compiled) + finally + close(table) + end + return bytes +end + +function n5productionbytecheck(case, pageversion::Symbol) + source = N5.n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=pageversion) + sourcedecoded = N5.n5decodefile(source) + table = Parquet.Table(source) + try + outputs = N5.n5productionencodedbytes(table, pageversion) + @test outputs.privatefirst == outputs.privatesecond + @test outputs.publicfirst == outputs.publicsecond + @test outputs.privatefirst == outputs.publicfirst + for bytes in (outputs.privatefirst, outputs.publicfirst) + decoded = N5.n5decodefile(bytes) + @test decoded.metadata.schema == case.schema + @test N5.n5schemaexact(decoded.metadata.schema, + sourcedecoded.metadata.schema) + @test decoded.metadata.num_rows == length(case.rows) + @test [leaf.path for leaf in decoded.leaves] == case.paths + @test all(==(pageversion), decoded.pageversions) + @test isequal(decoded.streams, case.streams) + compiled = N5.n5compile(case.node) + @test isequal(N5.n5assemble(compiled, decoded.streams, + length(case.rows)), case.rows) + end + return outputs.privatefirst + finally + close(table) + end +end + +function n5tableerror(bytes) + table = try + Parquet.Table(bytes) + catch error + return error + end + close(table) + return nothing +end + +function n5replace(value; changes...) + names = fieldnames(typeof(value)) + fields = map(names) do name + return haskey(changes, name) ? changes[name] : getfield(value, name) + end + return typeof(value)(fields...) +end + +function n5headermutator(pageversion::Symbol; changes...) + return function(header) + if pageversion === :v1 + data = n5replace(header.data_page_header; changes...) + return n5replace(header; data_page_header=data) + end + data = n5replace(header.data_page_header_v2; changes...) + return n5replace(header; data_page_header_v2=data) + end +end + +function n5pageheadermutator(; changes...) + return header -> n5replace(header; changes...) +end + +function n5footercolumnmutator(bytes::Vector{UInt8}; changes...) + metadata, dataend = N5.n5decodefooter(bytes) + rowgroup = only(metadata.row_groups) + chunk = only(rowgroup.columns) + column = n5replace(chunk.meta_data; changes...) + updatedchunk = n5replace(chunk; meta_data=column) + updatedgroup = n5replace(rowgroup; + columns=N5.MD.ColumnChunk[updatedchunk]) + updatedmetadata = n5replace(metadata; + row_groups=N5.MD.RowGroup[updatedgroup]) + footer = N5.TH.encode(updatedmetadata) + output = Vector{UInt8}(view(bytes, firstindex(bytes):dataend)) + append!(output, footer) + N5._n5pushu32!(output, UInt32(length(footer))) + append!(output, N5.N5_MAGIC) + return output +end + +function n5footerlengthmutator(bytes::Vector{UInt8}, lengthvalue::UInt32) + output = copy(bytes) + encoded = UInt8[] + N5._n5pushu32!(encoded, lengthvalue) + output[(end - 7):(end - 4)] = encoded + return output +end + +function n5modeldecodeerror(bytes::Vector{UInt8}) + try + N5.n5decodefile(bytes) + catch error + return error + end + return nothing +end + +function n5propertycodecenum(codec::Symbol) + codec === :uncompressed && return N5.MD.CompressionCodec.UNCOMPRESSED + codec === :snappy && return N5.MD.CompressionCodec.SNAPPY + codec === :gzip && return N5.MD.CompressionCodec.GZIP + codec === :brotli && return N5.MD.CompressionCodec.BROTLI + codec === :zstd && return N5.MD.CompressionCodec.ZSTD + codec === :lz4_raw && return N5.MD.CompressionCodec.LZ4_RAW + throw(ArgumentError("unsupported N5 property codec $codec")) +end + +function n5propertycodecmetadata(table, codec::Symbol) + expected = n5propertycodecenum(codec) + for rowgroup in table.metadata.row_groups + for chunk in rowgroup.columns + @test chunk.meta_data.codec == expected + end + end + return +end + +@testset "N5 independent binding goldens" begin + goldens = N5.n5bindinggoldens() + @test length(goldens) == 12 + @test [case.name for case in goldens] == [ + "list-rule-1", + "list-rule-2", + "list-rule-3", + "list-rule-4-array", + "list-rule-4-tuple", + "list-rule-5-required", + "list-rule-5-paired", + "list-rule-5-extended", + "direct-list-of-map", + "map-standard", + "map-key-only", + "map-optional-key", + ] + compiled = Dict{String,N5.N5Compiled}() + for case in goldens + compiled[case.name] = n5checkcase(case) + end + rule4 = only(case for case in goldens if case.name == "list-rule-4-array") + rule5 = only(case for case in goldens if case.name == "list-rule-5-paired") + @test isequal(rule4.streams, rule5.streams) + @test !isequal(rule4.rows, rule5.rows) + standard = only(case for case in goldens if case.name == "map-standard") + @test N5.n5maplookup(standard.rows[4], "a") == Int32(2) + @test_throws KeyError N5.n5maplookup(standard.rows[4], "absent") + optionalkey = only(case for case in goldens if + case.name == "map-optional-key") + key = optionalkey.streams[1] + definitions = copy(key.definition) + definitions[3] = UInt64(2) + mutated = copy(optionalkey.streams) + mutated[1] = N5.N5LeafStream(copy(key.repetition), definitions, + Any[key.values[2:end]...], key.max_repetition, key.max_definition) + @test_throws ArgumentError N5.n5assemble(compiled[optionalkey.name], + mutated, length(optionalkey.rows)) + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(optionalkey.schema, mutated, + length(optionalkey.rows); pageversion=pageversion) + @test n5tableerror(bytes) isa Parquet.FormatError + end +end + +@testset "N5 independent hybrid decoder" begin + levels = UInt64[0, 0, 1, 1, 1, 2, 2, 0] + encoded = N5.n5encodehybrid(levels, 3; length_prefix=false) + decoded, position = N5.n5decodehybrid(encoded, 1, length(encoded), + length(levels), 3; length_prefix=false) + @test decoded == levels + @test position == length(encoded) + 1 + prefixed = N5.n5encodehybrid(levels, 3; length_prefix=true) + decoded, position = N5.n5decodehybrid(prefixed, 1, length(prefixed), + length(levels), 3; length_prefix=true) + @test decoded == levels + @test position == length(prefixed) + 1 + bitpacked = UInt8[0x03, 0xe4, 0xe4] + expected = UInt64[0, 1, 2, 3, 0, 1, 2, 3] + decoded, position = N5.n5decodehybrid(bitpacked, 1, length(bitpacked), + 8, 3; length_prefix=false) + @test decoded == expected + @test position == length(bitpacked) + 1 + bitpackedprefix = UInt8[0x03, 0x00, 0x00, 0x00, bitpacked...] + decoded, position = N5.n5decodehybrid(bitpackedprefix, 1, + length(bitpackedprefix), 8, 3; length_prefix=true) + @test decoded == expected + @test position == length(bitpackedprefix) + 1 + @test_throws ArgumentError N5.n5decodehybrid(UInt8[0x02], 1, 1, + 1, 1; length_prefix=false) + @test_throws ArgumentError N5.n5decodehybrid(UInt8[0x02, 0x02], 1, 2, + 1, 1; length_prefix=false) + @test_throws ArgumentError N5.n5decodehybrid(UInt8[], 1, 0, + N5.N5_MAX_DECODE_VALUES + 1, 0; length_prefix=false) + oversizedrun = UInt8[fill(UInt8(0xff), 9)..., UInt8(0x01)] + error = try + N5.n5decodehybrid(oversizedrun, 1, length(oversizedrun), 1, 1; + length_prefix=false) + nothing + catch caught + caught + end + @test error isa ArgumentError + @test error.msg == "N5 hybrid bit-packed value count overflows Int" + error = try + N5.n5bitwidth(big(1) << 100) + nothing + catch caught + caught + end + @test error isa ArgumentError + @test error.msg == "N5 level maximum does not fit UInt64" +end + +@testset "N5 hostile serialized page-header counts" begin + case = first(N5.n5bindinggoldens()) + v1changes = ( + (; num_values=Int32(-1)), + (; num_values=Int32(N5.N5_MAX_DECODE_VALUES + 1)), + (; repetition_level_encoding=N5.MD.Encoding.BIT_PACKED), + (; definition_level_encoding=N5.MD.Encoding.BIT_PACKED), + ) + for changes in v1changes + bytes = N5.n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=:v1, headermutator=n5headermutator(:v1; changes...)) + @test_throws ArgumentError N5.n5decodefile(bytes) + end + v2changes = ( + (; num_values=Int32(-1)), + (; num_values=Int32(N5.N5_MAX_DECODE_VALUES + 1)), + (; num_rows=Int32(-1)), + (; num_rows=Int32(N5.N5_MAX_DECODE_VALUES + 1)), + (; num_nulls=Int32(-1)), + (; num_nulls=Int32(N5.N5_MAX_DECODE_VALUES + 1)), + (; repetition_levels_byte_length=Int32(-1)), + (; repetition_levels_byte_length=typemax(Int32)), + (; definition_levels_byte_length=Int32(-1)), + (; definition_levels_byte_length=typemax(Int32)), + ) + for changes in v2changes + bytes = N5.n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=:v2, headermutator=n5headermutator(:v2; changes...)) + @test_throws ArgumentError N5.n5decodefile(bytes) + end +end + +@testset "N5 hostile serialized footer and frame ranges" begin + case = first(N5.n5bindinggoldens()) + source = N5.n5emitfile(case.schema, case.streams, length(case.rows)) + footerchanges = ( + ((; data_page_offset=typemax(Int64), total_compressed_size=Int64(1)), + "N5 page first byte overflows Int"), + ((; data_page_offset=Int64(4), + total_compressed_size=typemax(Int64)), + "N5 page frame exceeds its bound"), + ((; data_page_offset=Int64(-1), total_compressed_size=Int64(1)), + "N5 page offset is negative"), + ((; data_page_offset=Int64(4), total_compressed_size=Int64(0)), + "N5 page frame is empty"), + ) + for (changes, message) in footerchanges + bytes = n5footercolumnmutator(source; changes...) + error = n5modeldecodeerror(bytes) + @test error isa ArgumentError + @test error.msg == message + end + for (lengthvalue, message) in ( + (UInt32(0), "N5 footer is empty"), + (typemax(UInt32), "N5 footer starts before data")) + error = n5modeldecodeerror(n5footerlengthmutator(source, lengthvalue)) + @test error isa ArgumentError + @test error.msg == message + end + pagechanges = ( + ((; compressed_page_size=Int32(-1), + uncompressed_page_size=Int32(-1)), + "N5 compressed page size is negative"), + ((; uncompressed_page_size=Int32(-1)), + "N5 uncompressed page size is negative"), + ((; compressed_page_size=typemax(Int32), + uncompressed_page_size=typemax(Int32)), + "N5 page payload exceeds its bound"), + ) + for pageversion in (:v1, :v2), (changes, message) in pagechanges + bytes = N5.n5emitfile(case.schema, case.streams, length(case.rows); + pageversion=pageversion, + headermutator=n5pageheadermutator(; changes...)) + error = n5modeldecodeerror(bytes) + @test error isa ArgumentError + @test error.msg == message + end +end + +@testset "N5 independent V1 and V2 fixtures" begin + for case in N5.n5bindinggoldens() + compiled = N5.n5compile(case.node) + v1 = n5checkbytes(case, compiled, :v1) + v2 = n5checkbytes(case, compiled, :v2) + @test v1 != v2 + end +end + +@testset "N5 independently decoded production writes" begin + for case in N5.n5bindinggoldens() + v1 = n5productionbytecheck(case, :v1) + v2 = n5productionbytecheck(case, :v2) + @test v1 != v2 + end +end + +@testset "N5 exact schema-bearing rewrite" begin + case = N5.n5provenancegolden() + @test !isempty(case.schema[1].unknown_fields) + @test !isempty(case.schema[2].unknown_fields) + @test !isempty(case.schema[2].logicalType.LIST.unknown_fields) + @test !isempty(case.schema[3].unknown_fields) + @test !isempty(case.schema[3].logicalType.unknown_fields) + for pageversion in (:v1, :v2) + n5productionbytecheck(case, pageversion) + end +end + +@testset "N5 checked golden manifest" begin + manifest = N5.n5goldenmanifest() + expected = N5.N5_GOLDEN_FILE_SHA256 + @test length(manifest) == length(expected) == 12 + for (entry, hashes) in zip(manifest, expected) + @test (entry.name, entry.v1_sha256, entry.v2_sha256) == hashes + end + firstbytes = N5.n5encodemanifest(manifest) + @test firstbytes == N5.n5encodemanifest(N5.n5goldenmanifest()) + @test N5.n5manifestsha256(manifest) == + N5.N5_GOLDEN_MANIFEST_SHA256 +end + +@testset "N5 production boundaries for binding goldens" begin + goldens = N5.n5bindinggoldens() + for case in goldens + for pageversion in (:v1, :v2) + n5productioncheck(case.node, case.schema, case.streams, + case.rows, pageversion) + end + end + standard = only(case for case in goldens if case.name == "map-standard") + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(standard.schema, standard.streams, + length(standard.rows); pageversion=pageversion) + table = Parquet.Table(bytes) + try + duplicated = table.columns.attrs[4] + @test Parquet.maplookup(duplicated, "a") == Int32(2) + @test_throws KeyError Parquet.maplookup(duplicated, "absent") + finally + close(table) + end + end +end + +@testset "N5 complete accepted MAP Cartesian product" begin + cases = N5.n5mapmatrix() + @test length(cases) == 1152 + @test Set(case.node.marker for case in cases) == Set((:modern, :dual, + :modern_alias, :conflict, :modern_primitive, :modern_unknown, + :legacy, :alias)) + @test Set(case.node.entrymarker for case in cases) == Set((:none, :marked, + :empty, :future, :future_marked, :unknown_converted)) + seen = Set{String}() + for case in cases + @test case.name ∉ seen + push!(seen, case.name) + compiled = N5.n5compile(case.node) + schema = N5.n5schema(case.node) + streams = N5.n5shred(compiled, case.rows) + @test isequal(N5.n5assemble(compiled, streams, length(case.rows)), + case.rows) + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(schema, streams, length(case.rows); + pageversion=pageversion) + decoded = N5.n5decodefile(bytes) + @test decoded.metadata.schema == schema + @test isequal(decoded.streams, streams) + @test isequal(N5.n5assemble(compiled, decoded.streams, + length(case.rows)), case.rows) + n5productioncheck(case.node, schema, streams, case.rows, + pageversion) + end + end +end + +@testset "N5 recursive MAP key and value shapes" begin + cases = N5.n5recursivecases() + @test length(cases) == 3 + for case in cases + compiled = N5.n5compile(case.node) + schema = N5.n5schema(case.node) + streams = N5.n5shred(compiled, case.rows) + @test isequal(N5.n5assemble(compiled, streams, length(case.rows)), + case.rows) + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(schema, streams, length(case.rows); + pageversion=pageversion) + decoded = N5.n5decodefile(bytes) + @test decoded.metadata.schema == schema + @test isequal(decoded.streams, streams) + n5productioncheck(case.node, schema, streams, case.rows, + pageversion) + end + end +end + +@testset "N5 annotation precedence controls" begin + for case in N5.n5annotationcontrols() + compiled = N5.n5compile(case.node) + schema = N5.n5schema(case.node) + streams = N5.n5shred(compiled, case.rows) + @test isequal(N5.n5assemble(compiled, streams, length(case.rows)), + case.rows) + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(schema, streams, length(case.rows); + pageversion=pageversion) + decoded = N5.n5decodefile(bytes) + @test decoded.metadata.schema == schema + @test isequal(decoded.streams, streams) + n5productioncheck(case.node, schema, streams, case.rows, + pageversion) + end + end +end + +@testset "N5 serialized LIST precedence rows" begin + controls = N5.n5listbindingcontrols() + @test [control.name for control in controls] == [ + "list-rule-2-multifield-array", + "list-rule-3-repeated-array", + "list-rule-3-unannotated-group", + "list-rule-5-Array", + "list-rule-5-ARRAY", + "list-rule-5-case-mismatched-tuple", + "list-unknown-blocks-legacy", + ] + for control in controls + @test control.node !== nothing + node = control.node + compiled = N5.n5compile(node) + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(control.schema, control.streams, + length(control.rows); pageversion=pageversion) + decoded = N5.n5decodefile(bytes) + @test decoded.metadata.schema == control.schema + @test isequal(decoded.streams, control.streams) + table = Parquet.Table(bytes) + try + column = first(values(table.columns)) + if control.expected === :list + @test column isa Parquet.ListVector + else + @test column isa Parquet.StructVector + end + @test isequal(N5.n5normalizetable(node, table), control.rows) + expectedtree = N5.n5expectedvectortree(node, control.rows) + @test N5.n5comparevectortree(expectedtree, table.columns) + actual, fields, rows = N5.n5productionstreams(table, + compiled) + @test rows == length(control.rows) + @test fields.elements == control.schema + @test N5.n5compareproductionstreams(actual, + control.streams, compiled) + finally + close(table) + end + end + end +end + +@testset "N5 complete MAP annotation binding rows" begin + controls = N5.n5mapbindingcontrols() + @test [control.name for control in controls] == [ + "outer-modern-map", + "outer-modern-map-matching", + "outer-modern-map-alias", + "outer-modern-map-conflict", + "outer-modern-map-primitive", + "outer-modern-map-unknown-converted", + "outer-legacy-map", + "outer-legacy-map-alias", + "outer-unknown-blocks-map", + "outer-unknown-blocks-alias", + "outer-unannotated", + "outer-unknown-converted", + "outer-list-wins-map", + "outer-list-wins-alias", + "outer-variant-wins", + "outer-empty-wins", + "outer-converted-list", + "entry-unmarked", + "entry-map-key-value", + "entry-empty-logical", + "entry-unknown-logical", + "entry-unknown-blocks-map-key-value", + "entry-unknown-converted", + ] + for control in controls + @test control.node !== nothing + node = control.node + compiled = N5.n5compile(node) + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(control.schema, control.streams, + length(control.rows); pageversion=pageversion) + decoded = N5.n5decodefile(bytes) + @test decoded.metadata.schema == control.schema + @test isequal(decoded.streams, control.streams) + table = Parquet.Table(bytes) + try + column = first(values(table.columns)) + if control.expected === :map + @test column isa Parquet.MapVector + elseif control.expected === :list + @test column isa Parquet.ListVector + else + @test column isa Parquet.StructVector + end + @test isequal(N5.n5normalizetable(node, table), control.rows) + expectedtree = N5.n5expectedvectortree(node, control.rows) + @test N5.n5comparevectortree(expectedtree, table.columns) + actual, fields, rows = N5.n5productionstreams(table, + compiled) + @test rows == length(control.rows) + @test fields.elements == control.schema + @test N5.n5compareproductionstreams(actual, + control.streams, compiled) + finally + close(table) + end + end + end +end + +@testset "N5 complete MAP rejected binding neighbors" begin + failures = N5.n5mapbindingfailures() + @test [failure.name for failure in failures] == [ + "entry-logical-map", + "entry-logical-list", + "entry-logical-variant", + "entry-logical-primitive", + "entry-converted-map", + "entry-converted-list", + "entry-converted-primitive", + "outer-primitive-logical", + "outer-primitive-converted", + "outer-repeated-outside-list", + "outer-zero-child", + "outer-two-children", + "entry-primitive", + "entry-non-repeated", + "entry-zero-child-key-absent", + "entry-three-children", + "key-repeated", + "value-repeated", + ] + for failure in failures + for pageversion in (:v1, :v2) + bytes = N5.n5emitfile(failure.schema, failure.streams, 0; + pageversion=pageversion) + @test n5tableerror(bytes) isa Parquet.FormatError + end + end +end + +@testset "N5 model validation" begin + @test_throws ArgumentError N5.n5list("bad", + N5.n5primitive("value", N5.MD.Type.INT32; optional=true); + layout=:rule1) + @test_throws ArgumentError N5.n5list("bad", + N5.n5primitive("value", N5.MD.Type.INT32); layout=:rule2) + @test_throws ArgumentError N5.n5list("bad", + N5.n5primitive("value", N5.MD.Type.INT32); layout=:rule3) + @test_throws ArgumentError N5.n5struct("empty", N5.N5Node[]) + required = N5.n5primitive("value", N5.MD.Type.INT32) + @test_throws ArgumentError N5.n5shred(N5.n5compile(required), Any[missing]) + key = N5.n5primitive("key", N5.MD.Type.BYTE_ARRAY; + optional=true, logical=:string) + mapnode = N5.n5map("attrs", key, nothing) + @test_throws ArgumentError N5.n5shred(N5.n5compile(mapnode), + Any[N5.N5MapValue(N5.N5Entry[ N5.N5Entry(missing, nothing, false) ])]) + bytes = N5.n5emitfile(N5.n5schema(required), + N5.n5shred(N5.n5compile(required), Any[Int32(1)]), 1) + error = try + N5.n5emitfile(N5.n5schema(required), + N5.n5shred(N5.n5compile(required), Any[Int32(1)]), + typemax(UInt128)) + nothing + catch caught + caught + end + @test error isa ArgumentError + @test error.msg == "N5 row count does not fit Int" + corrupted = copy(bytes) + corrupted[1] = 0x00 + @test_throws ArgumentError N5.n5decodefile(corrupted) + truncated = bytes[1:(end - 1)] + @test_throws ArgumentError N5.n5decodefile(truncated) +end + +@testset "N5 deterministic recursive property schedule" begin + cases = N5_PROPERTY_CASES + rejected = N5_PROPERTY_REJECTED + @test length(cases) == N5.N5_PROPERTY_CASE_COUNT == 256 + @test rejected == length(N5.N5_PROPERTY_REJECTED_PREFIX) == 6 + @test first(cases).id == 0 + @test last(cases).id == 261 + @test issorted(case.id for case in cases) + rejectedids = [id for (id, _) in N5.N5_PROPERTY_REJECTED_PREFIX] + @test [id for id in 0:last(cases).id if + all(case -> case.id != id, cases)] == rejectedids + @test [case.pageversion for case in cases] == + [isodd(index) ? :v1 : :v2 for index in eachindex(cases)] + for (id, reason) in N5.N5_PROPERTY_REJECTED_PREFIX + assessment = N5.n5propertycandidateassessment(id) + @test assessment.candidate === nothing + @test assessment.reason === reason + actual = getproperty(assessment.metrics, reason) + limit = reason === :depth ? 6 : reason === :width ? 4 : + reason === :astnodes ? N5.N5_PROPERTY_MAX_AST_NODES : + reason === :leaves ? N5.N5_PROPERTY_MAX_LEAVES : + reason === :levelentries ? N5.N5_PROPERTY_MAX_LEVEL_ENTRIES : + reason === :payloadbytes ? N5.N5_PROPERTY_MAX_PAYLOAD_BYTES : 0 + @test actual > limit + end + after = N5.n5propertycandidateassessment(last(cases).id + 1) + @test after.reason === nothing + repeated, repeatedrejected = N5.n5propertycases() + @test repeatedrejected == rejected + @test [case.id for case in repeated] == [case.id for case in cases] + @test last(repeated).id < N5.N5_PROPERTY_CANDIDATE_LAST + coverage = N5.n5propertycoverage(cases, rejected) + @test isempty(N5.n5propertymissingcoverage(coverage)) + @test coverage.rejected_candidates == rejected + manifest = N5.n5encodepropertymanifest(cases, rejected) + @test manifest == N5.n5encodepropertymanifest(repeated, repeatedrejected) + @test N5.n5propertymanifestsha256(cases, rejected) == + N5.N5_PROPERTY_MANIFEST_SHA256 + @test N5.n5manifestsha256(N5.n5goldenmanifest()) == + N5.N5_GOLDEN_MANIFEST_SHA256 +end + +@testset "N5 explicit zero-row property files" begin + cases = [case for case in N5_PROPERTY_CASES if isempty(case.rows)] + @test length(cases) == 11 + for case in cases, pageversion in (:v1, :v2) + bytes = N5.n5emitfile(case.schema, case.streams, 0; + pageversion=pageversion) + decoded = N5.n5decodefile(bytes) + @test decoded.metadata.num_rows == 0 + @test isempty(decoded.metadata.row_groups) + @test isempty(decoded.pageversions) + @test all(isempty(stream.repetition) && + isempty(stream.definition) && isempty(stream.values) + for stream in decoded.streams) + table = Parquet.Table(bytes) + try + @test table.rows == 0 + @test isempty(table.metadata.row_groups) + @test N5.n5schemaexact(table.metadata.schema, + decoded.metadata.schema) + @test isequal(N5.n5normalizetable(case.node, table), case.rows) + expectedtree = N5.n5expectedvectortree(case.node, case.rows) + @test N5.n5comparevectortree(expectedtree, table.columns) + finally + close(table) + end + end +end + +@testset "N5 256 independent and production property cases" begin + for case in N5_PROPERTY_CASES + @testset "$(N5.n5propertydiagnostic(case))" begin + @test 1 <= case.depth <= 6 + @test 1 <= case.width <= 4 + @test case.astnodes <= N5.N5_PROPERTY_MAX_AST_NODES + @test case.leaves <= N5.N5_PROPERTY_MAX_LEAVES + @test case.levelentries <= N5.N5_PROPERTY_MAX_LEVEL_ENTRIES + @test case.densevalues <= N5.N5_PROPERTY_MAX_DENSE_VALUES + @test case.payloadbytes <= N5.N5_PROPERTY_MAX_PAYLOAD_BYTES + @test all(count(iszero, stream.repetition) == length(case.rows) + for stream in case.streams) + compiled = n5checkcase(case) + independent = n5checkbytes(case, compiled, case.pageversion) + @test independent == N5.n5emitfile(case.schema, case.streams, + length(case.rows); pageversion=case.pageversion) + @test n5productioncheck(case.node, case.schema, case.streams, + case.rows, case.pageversion) == independent + production = n5productionbytecheck(case, case.pageversion) + source = N5.n5decodefile(independent) + table = Parquet.Table(production) + try + @test N5.n5schemaexact(table.metadata.schema, + source.metadata.schema) + @test isequal(N5.n5normalizetable(case.node, table), case.rows) + expectedtree = N5.n5expectedvectortree(case.node, case.rows) + @test N5.n5comparevectortree(expectedtree, table.columns) + finally + close(table) + end + end + end +end + +@testset "N5 stable six-codec property subset" begin + cases = N5_PROPERTY_CASES + subset = N5.n5propertycodecsubset(cases) + @test Tuple(case.id for case in subset) == N5.N5_PROPERTY_CODEC_CASE_IDS + @test length(Set(case.pageversion for case in subset)) == 2 + fixtures = N5.n5propertycodecfixtures(cases) + @test length(fixtures) == + N5.N5_PROPERTY_CODEC_COUNT * length(N5.N5_PROPERTY_CODECS) == 192 + @test length(Set(fixture.filename for fixture in fixtures)) == + length(fixtures) + byid = Dict(case.id => case for case in subset) + seen = Set{Tuple{Int,Symbol}}() + for fixture in fixtures + case = byid[fixture.caseid] + push!(seen, (fixture.caseid, fixture.codec)) + @test fixture.name == case.name + @test fixture.pageversion == case.pageversion + @test fixture.schema == case.schema + @test fixture.paths == case.paths + @test isequal(fixture.rows, case.rows) + @test fixture.sha256 == bytes2hex(SHA.sha256(fixture.bytes)) + table = Parquet.Table(fixture.bytes) + try + @test table.rows == length(case.rows) + @test N5.n5schemaexact(table.metadata.schema, fixture.schema) + @test isequal(N5.n5normalizetable(case.node, table), case.rows) + expectedtree = N5.n5expectedvectortree(case.node, case.rows) + @test N5.n5comparevectortree(expectedtree, table.columns) + actual, fields, rows = N5.n5productionstreams(table, + N5.n5compile(case.node)) + @test rows == length(case.rows) + @test N5.n5schemaexact(fields.elements, fixture.schema) + @test N5.n5compareproductionstreams(actual, case.streams, + N5.n5compile(case.node)) + n5propertycodecmetadata(table, fixture.codec) + finally + close(table) + end + end + @test seen == Set((case.id, codec) for case in subset + for codec in N5.N5_PROPERTY_CODECS) +end diff --git a/test/conformance/n5/model/wire.jl b/test/conformance/n5/model/wire.jl new file mode 100644 index 0000000..0ece048 --- /dev/null +++ b/test/conformance/n5/model/wire.jl @@ -0,0 +1,652 @@ +const N5_MAGIC = UInt8[0x50, 0x41, 0x52, 0x31] +const N5_MAX_DECODE_VALUES = 1_048_576 + +struct N5DecodedFile + metadata::MD.FileMetaData + leaves::Vector{N5PhysicalLeaf} + streams::Vector{N5LeafStream} + pageversions::Vector{Symbol} +end + +function _n5checkedint(value::Integer, label::String) + try + return Int(value) + catch error + error isa InexactError || error isa OverflowError || rethrow() + throw(ArgumentError("N5 $label does not fit Int")) + end +end + +function _n5checkeduint64(value::Integer, label::String) + try + return UInt64(value) + catch error + error isa InexactError || error isa OverflowError || rethrow() + throw(ArgumentError("N5 $label does not fit UInt64")) + end +end + +function _n5checkedadd(left::Int, right::Int, label::String) + try + return Base.checked_add(left, right) + catch error + error isa OverflowError || rethrow() + throw(ArgumentError("N5 $label overflows Int")) + end +end + +function _n5checkedmul(left::Int, right::Int, label::String) + try + return Base.checked_mul(left, right) + catch error + error isa OverflowError || rethrow() + throw(ArgumentError("N5 $label overflows Int")) + end +end + +function _n5inputlast(bytes::AbstractVector{UInt8}, last::Int, label::String) + first = firstindex(bytes) + minimumlast = first - 1 + minimumlast <= last <= lastindex(bytes) || throw(ArgumentError( + "N5 $label is outside the input")) + return last +end + +function _n5rangelast(position::Int, lengthvalue::Int, limit::Int, label::String) + position >= 1 || throw(ArgumentError("N5 $label starts before input")) + lengthvalue >= 0 || throw(ArgumentError("N5 $label length is negative")) + limit >= 0 || throw(ArgumentError("N5 $label bound is before input")) + if iszero(lengthvalue) + position <= limit || (limit < typemax(Int) && position == limit + 1) || + throw(ArgumentError("N5 $label starts after its bound")) + return position - 1 + end + position <= limit || throw(ArgumentError("N5 $label starts after its bound")) + lengthvalue - 1 <= limit - position || throw(ArgumentError( + "N5 $label exceeds its bound")) + return position + lengthvalue - 1 +end + +function _n5framerange(bytes::AbstractVector{UInt8}, offset::Integer, + framesize::Integer, limit::Int) + offset >= 0 || throw(ArgumentError("N5 page offset is negative")) + framesize > 0 || throw(ArgumentError("N5 page frame is empty")) + _n5inputlast(bytes, limit, "page frame bound") + offsetvalue = _n5checkedint(offset, "page offset") + framevalue = _n5checkedint(framesize, "page frame size") + first = _n5checkedadd(offsetvalue, 1, "page first byte") + last = _n5rangelast(first, framevalue, limit, "page frame") + return first, last +end + +function _n5pushu32!(bytes::Vector{UInt8}, value::UInt32) + for shift in (0, 8, 16, 24) + push!(bytes, UInt8((value >> shift) & 0xff)) + end + return +end + +function _n5pushu64!(bytes::Vector{UInt8}, value::UInt64) + for shift in (0, 8, 16, 24, 32, 40, 48, 56) + push!(bytes, UInt8((value >> shift) & 0xff)) + end + return +end + +function _n5readu32(bytes::AbstractVector{UInt8}, position::Int, last::Int) + _n5inputlast(bytes, last, "UInt32 frame") + position >= firstindex(bytes) || throw(ArgumentError( + "N5 UInt32 starts before input")) + _n5rangelast(position, 4, last, "UInt32") + value = UInt32(0) + for offset in 0:3 + value |= UInt32(bytes[position + offset]) << (8 * offset) + end + return value, _n5checkedadd(position, 4, "UInt32 cursor") +end + +function _n5readu64(bytes::AbstractVector{UInt8}, position::Int, last::Int) + _n5inputlast(bytes, last, "UInt64 frame") + position >= firstindex(bytes) || throw(ArgumentError( + "N5 UInt64 starts before input")) + _n5rangelast(position, 8, last, "UInt64") + value = UInt64(0) + for offset in 0:7 + value |= UInt64(bytes[position + offset]) << (8 * offset) + end + return value, _n5checkedadd(position, 8, "UInt64 cursor") +end + +function _n5pushuleb!(bytes::Vector{UInt8}, value::UInt64) + current = value + while current >= 0x80 + push!(bytes, UInt8(current & 0x7f) | 0x80) + current >>= 7 + end + push!(bytes, UInt8(current)) + return +end + +function _n5readuleb(bytes::AbstractVector{UInt8}, position::Int, last::Int) + _n5inputlast(bytes, last, "hybrid header frame") + position >= firstindex(bytes) || throw(ArgumentError( + "N5 hybrid header starts before input")) + value = UInt64(0) + shift = 0 + cursor = position + for _ in 1:10 + cursor <= last || throw(ArgumentError("N5 hybrid header is truncated")) + byte = bytes[cursor] + cursor += 1 + shift == 63 && byte > 0x01 && throw(ArgumentError( + "N5 hybrid header overflows UInt64")) + value |= UInt64(byte & 0x7f) << shift + byte & 0x80 == 0 && return value, cursor + shift += 7 + end + throw(ArgumentError("N5 hybrid header is too long")) +end + +function n5bitwidth(maximum::Integer) + maximum >= 0 || throw(ArgumentError("N5 level maximum is negative")) + iszero(maximum) && return 0 + maximumvalue = _n5checkeduint64(maximum, "level maximum") + return 64 - leading_zeros(maximumvalue) +end + +function _n5encoderuns(levels::AbstractVector{UInt64}, bitwidth::Int) + isempty(levels) && return UInt8[] + bytes = UInt8[] + width = cld(bitwidth, 8) + index = firstindex(levels) + while index <= lastindex(levels) + value = levels[index] + (bitwidth == 64 || value < (UInt64(1) << bitwidth)) || + throw(ArgumentError( + "N5 level $value does not fit bit width $bitwidth")) + stop = index + while stop < lastindex(levels) && levels[stop + 1] == value + stop += 1 + end + count = stop - index + 1 + _n5pushuleb!(bytes, UInt64(count) << 1) + for offset in 0:(width - 1) + push!(bytes, UInt8((value >> (8 * offset)) & 0xff)) + end + index = stop + 1 + end + return bytes +end + +function n5encodehybrid(levels::AbstractVector{UInt64}, maximum::Integer; + length_prefix::Bool) + bitwidth = n5bitwidth(maximum) + bitwidth == 0 && return UInt8[] + payload = _n5encoderuns(levels, bitwidth) + length(payload) <= typemax(UInt32) || throw(ArgumentError( + "N5 hybrid payload exceeds UInt32")) + length_prefix || return payload + output = UInt8[] + _n5pushu32!(output, UInt32(length(payload))) + append!(output, payload) + return output +end + +function _n5rlevalue(bytes::AbstractVector{UInt8}, position::Int, last::Int, + width::Int) + _n5inputlast(bytes, last, "hybrid RLE frame") + _n5rangelast(position, width, last, "hybrid RLE value") + value = UInt64(0) + for offset in 0:(width - 1) + value |= UInt64(bytes[position + offset]) << (8 * offset) + end + return value, _n5checkedadd(position, width, "hybrid RLE cursor") +end + +function _n5bitpackedvalue(bytes::AbstractVector{UInt8}, position::Int, + bitoffset::Int, bitwidth::Int) + value = UInt64(0) + for bit in 0:(bitwidth - 1) + absolute = bitoffset + bit + byte = bytes[position + (absolute >> 3)] + value |= UInt64((byte >> (absolute & 7)) & 0x01) << bit + end + return value +end + +function n5decodehybrid(bytes::AbstractVector{UInt8}, position::Int, last::Int, + count::Integer, maximum::Integer; length_prefix::Bool) + count >= 0 || throw(ArgumentError("N5 hybrid count is negative")) + count <= N5_MAX_DECODE_VALUES || throw(ArgumentError( + "N5 hybrid count exceeds the focused decoder limit")) + requested = Int(count) + bitwidth = n5bitwidth(maximum) + bitwidth == 0 && return fill(UInt64(0), requested), position + _n5inputlast(bytes, last, "hybrid frame") + payloadlast = last + cursor = position + if length_prefix + lengthvalue, cursor = _n5readu32(bytes, cursor, last) + payloadlength = _n5checkedint(lengthvalue, "hybrid payload length") + payloadlast = _n5rangelast(cursor, payloadlength, last, + "hybrid length prefix") + end + output = UInt64[] + sizehint!(output, requested) + width = cld(bitwidth, 8) + while length(output) < requested + header, cursor = _n5readuleb(bytes, cursor, payloadlast) + if iszero(header & 0x01) + run = _n5checkedint(header >> 1, "hybrid RLE run length") + run > 0 || throw(ArgumentError("N5 hybrid RLE run is empty")) + value, cursor = _n5rlevalue(bytes, cursor, payloadlast, width) + value <= UInt64(maximum) || throw(ArgumentError( + "N5 hybrid level exceeds its maximum")) + append!(output, fill(value, min(run, requested - length(output)))) + else + groups = _n5checkedint(header >> 1, + "hybrid bit-packed group count") + groups > 0 || throw(ArgumentError("N5 hybrid bit-packed run is empty")) + values = _n5checkedmul(groups, 8, "hybrid bit-packed value count") + payloadbytes = _n5checkedmul(groups, bitwidth, + "hybrid bit-packed byte count") + _n5rangelast(cursor, payloadbytes, payloadlast, + "hybrid bit-packed run") + take = min(values, requested - length(output)) + for index in 0:(take - 1) + value = _n5bitpackedvalue(bytes, cursor, index * bitwidth, + bitwidth) + value <= UInt64(maximum) || throw(ArgumentError( + "N5 hybrid level exceeds its maximum")) + push!(output, value) + end + cursor = _n5checkedadd(cursor, payloadbytes, + "hybrid bit-packed cursor") + end + end + length_prefix && cursor != _n5checkedadd(payloadlast, 1, + "hybrid payload end") && throw(ArgumentError( + "N5 hybrid payload has trailing bytes")) + return output, cursor +end + +function _n5plainencode(values::Vector{Any}, leaf::N5PhysicalLeaf) + bytes = UInt8[] + physical = leaf.element.type_ + if physical == MD.Type.INT32 + for value in values + _n5pushu32!(bytes, reinterpret(UInt32, Int32(value))) + end + elseif physical == MD.Type.INT64 + for value in values + _n5pushu64!(bytes, reinterpret(UInt64, Int64(value))) + end + elseif physical == MD.Type.FLOAT + for value in values + _n5pushu32!(bytes, reinterpret(UInt32, Float32(value))) + end + elseif physical == MD.Type.DOUBLE + for value in values + _n5pushu64!(bytes, reinterpret(UInt64, Float64(value))) + end + elseif physical == MD.Type.BOOLEAN + for base in 1:8:length(values) + byte = UInt8(0) + for offset in 0:min(7, length(values) - base) + Bool(values[base + offset]) && (byte |= UInt8(1) << offset) + end + push!(bytes, byte) + end + elseif physical == MD.Type.BYTE_ARRAY + for value in values + data = value isa AbstractString ? codeunits(value) : value + length(data) <= typemax(Int32) || throw(ArgumentError( + "N5 byte array exceeds Int32")) + _n5pushu32!(bytes, reinterpret(UInt32, Int32(length(data)))) + append!(bytes, data) + end + else + throw(ArgumentError("unsupported N5 PLAIN physical type $physical")) + end + return bytes +end + +function _n5plainisstring(element::MD.SchemaElement) + logical = element.logicalType + logical !== nothing && logical.STRING !== nothing && return true + return element.converted_type == MD.ConvertedType.UTF8 +end + +function _n5plaindecode(bytes::AbstractVector{UInt8}, position::Int, last::Int, + count::Int, leaf::N5PhysicalLeaf) + values = [] + sizehint!(values, count) + cursor = position + physical = leaf.element.type_ + if physical == MD.Type.INT32 + for _ in 1:count + raw, cursor = _n5readu32(bytes, cursor, last) + push!(values, reinterpret(Int32, raw)) + end + elseif physical == MD.Type.INT64 + for _ in 1:count + raw, cursor = _n5readu64(bytes, cursor, last) + push!(values, reinterpret(Int64, raw)) + end + elseif physical == MD.Type.FLOAT + for _ in 1:count + raw, cursor = _n5readu32(bytes, cursor, last) + push!(values, reinterpret(Float32, raw)) + end + elseif physical == MD.Type.DOUBLE + for _ in 1:count + raw, cursor = _n5readu64(bytes, cursor, last) + push!(values, reinterpret(Float64, raw)) + end + elseif physical == MD.Type.BOOLEAN + needed = cld(count, 8) + _n5inputlast(bytes, last, "BOOLEAN frame") + _n5rangelast(cursor, needed, last, "BOOLEAN payload") + for index in 0:(count - 1) + byte = bytes[cursor + (index >> 3)] + push!(values, !iszero((byte >> (index & 7)) & 0x01)) + end + cursor = _n5checkedadd(cursor, needed, "BOOLEAN cursor") + elseif physical == MD.Type.BYTE_ARRAY + stringvalue = _n5plainisstring(leaf.element) + for _ in 1:count + rawlength, cursor = _n5readu32(bytes, cursor, last) + lengthvalue = Int(reinterpret(Int32, rawlength)) + lengthvalue >= 0 || throw(ArgumentError( + "N5 BYTE_ARRAY length is negative")) + datalast = _n5rangelast(cursor, lengthvalue, last, + "BYTE_ARRAY payload") + data = Vector{UInt8}(view(bytes, cursor:datalast)) + push!(values, stringvalue ? String(data) : data) + cursor = _n5checkedadd(cursor, lengthvalue, "BYTE_ARRAY cursor") + end + else + throw(ArgumentError("unsupported N5 PLAIN physical type $physical")) + end + return values, cursor +end + +function _n5page(stream::N5LeafStream, leaf::N5PhysicalLeaf, + pageversion::Symbol, headermutator::Function) + repetition = n5encodehybrid(stream.repetition, stream.max_repetition; + length_prefix=pageversion === :v1) + definition = n5encodehybrid(stream.definition, stream.max_definition; + length_prefix=pageversion === :v1) + values = _n5plainencode(stream.values, leaf) + payload = vcat(repetition, definition, values) + entries = length(stream.repetition) + entries <= typemax(Int32) || throw(ArgumentError("N5 page has too many entries")) + if pageversion === :v1 + data = MD.DataPageHeader(num_values=Int32(entries), + encoding=MD.Encoding.PLAIN, + definition_level_encoding=MD.Encoding.RLE, + repetition_level_encoding=MD.Encoding.RLE) + header = MD.PageHeader(type_=MD.PageType.DATA_PAGE, + uncompressed_page_size=Int32(length(payload)), + compressed_page_size=Int32(length(payload)), data_page_header=data) + elseif pageversion === :v2 + rows = count(iszero, stream.repetition) + nulls = count(!=(UInt64(stream.max_definition)), stream.definition) + data = MD.DataPageHeaderV2(num_values=Int32(entries), + num_nulls=Int32(nulls), num_rows=Int32(rows), + encoding=MD.Encoding.PLAIN, + definition_levels_byte_length=Int32(length(definition)), + repetition_levels_byte_length=Int32(length(repetition)), + is_compressed=false) + header = MD.PageHeader(type_=MD.PageType.DATA_PAGE_V2, + uncompressed_page_size=Int32(length(payload)), + compressed_page_size=Int32(length(payload)), + data_page_header_v2=data) + else + throw(ArgumentError("unsupported N5 page version $pageversion")) + end + header = headermutator(header) + header isa MD.PageHeader || throw(ArgumentError( + "N5 page-header mutator returned $(typeof(header))")) + headerbytes = TH.encode(header) + return vcat(headerbytes, payload), length(headerbytes) +end + +function _n5columnmetadata(stream::N5LeafStream, leaf::N5PhysicalLeaf, + offset::Int64, framesize::Int64, uncompressed::Int64, + pageversion::Symbol) + encodings = MD.Encoding.T[MD.Encoding.PLAIN] + (!iszero(stream.max_repetition) || !iszero(stream.max_definition)) && + pushfirst!(encodings, MD.Encoding.RLE) + pagetype = pageversion === :v1 ? MD.PageType.DATA_PAGE : + MD.PageType.DATA_PAGE_V2 + stats = MD.PageEncodingStats[MD.PageEncodingStats(page_type=pagetype, + encoding=MD.Encoding.PLAIN, count=Int32(1))] + return MD.ColumnMetaData(type_=leaf.element.type_, encodings=encodings, + path_in_schema=leaf.path, codec=MD.CompressionCodec.UNCOMPRESSED, + num_values=Int64(length(stream.repetition)), + total_uncompressed_size=uncompressed, + total_compressed_size=framesize, data_page_offset=offset, + encoding_stats=stats) +end + +function n5emitfile(schema::Vector{MD.SchemaElement}, + streams::Vector{N5LeafStream}, rows::Integer; + pageversion::Symbol=:v1, headermutator::Function=identity) + rows >= 0 || throw(ArgumentError("N5 row count is negative")) + rowcount = _n5checkedint(rows, "row count") + leaves = n5physicalleaves(schema) + length(leaves) == length(streams) || throw(ArgumentError( + "N5 schema and stream leaf counts differ")) + body = copy(N5_MAGIC) + if iszero(rowcount) + all(stream -> isempty(stream.repetition) && + isempty(stream.definition) && isempty(stream.values), streams) || + throw(ArgumentError("N5 zero-row streams are not empty")) + metadata = MD.FileMetaData(version=Int32(1), schema=schema, + num_rows=Int64(0), row_groups=MD.RowGroup[], + created_by="Parquet.jl N5 independent model") + footer = TH.encode(metadata) + append!(body, footer) + _n5pushu32!(body, UInt32(length(footer))) + append!(body, N5_MAGIC) + return body + end + chunks = MD.ColumnChunk[] + totaluncompressed = Int64(0) + totalcompressed = Int64(0) + for (leaf, stream) in zip(leaves, streams) + leaf.max_repetition == stream.max_repetition || throw(ArgumentError( + "N5 stream maximum repetition differs from schema")) + leaf.max_definition == stream.max_definition || throw(ArgumentError( + "N5 stream maximum definition differs from schema")) + count(iszero, stream.repetition) == rowcount || throw(ArgumentError( + "N5 stream row count differs")) + offset = Int64(length(body)) + frame, headerlength = _n5page(stream, leaf, pageversion, + headermutator) + append!(body, frame) + framesize = Int64(length(frame)) + uncompressed = Int64(headerlength) + + Int64(length(frame) - headerlength) + metadata = _n5columnmetadata(stream, leaf, offset, framesize, + uncompressed, pageversion) + push!(chunks, MD.ColumnChunk(file_offset=Int64(0), + meta_data=metadata)) + totaluncompressed += uncompressed + totalcompressed += framesize + end + rowgroup = MD.RowGroup(columns=chunks, + total_byte_size=totaluncompressed, num_rows=Int64(rowcount), + total_compressed_size=totalcompressed, file_offset=Int64(4), + ordinal=Int16(0)) + metadata = MD.FileMetaData(version=Int32(1), schema=schema, + num_rows=Int64(rowcount), row_groups=MD.RowGroup[rowgroup], + created_by="Parquet.jl N5 independent model") + footer = TH.encode(metadata) + append!(body, footer) + _n5pushu32!(body, UInt32(length(footer))) + append!(body, N5_MAGIC) + return body +end + +function n5decodefooter(bytes::AbstractVector{UInt8}) + length(bytes) >= 12 || throw(ArgumentError("N5 Parquet file is too short")) + bytes[1:4] == N5_MAGIC || throw(ArgumentError("N5 leading magic differs")) + bytes[(end - 3):end] == N5_MAGIC || throw(ArgumentError( + "N5 trailing magic differs")) + filelast = lastindex(bytes) + trailerstart = filelast - 7 + rawlength, _ = _n5readu32(bytes, trailerstart, filelast) + footerlength = _n5checkedint(rawlength, "footer length") + footerlength > 0 || throw(ArgumentError("N5 footer is empty")) + footerlast = filelast - 8 + footerlength <= footerlast - 4 || throw(ArgumentError( + "N5 footer starts before data")) + footerstart = footerlast - footerlength + 1 + reader = TH.Reader(bytes, footerstart, footerlast) + metadata = TH.decode(reader, MD.FileMetaData) + TH.consumed(reader) == footerlength || throw(ArgumentError( + "N5 footer has trailing Thrift bytes")) + return metadata, footerstart - 1 +end + +function _n5decodepage(bytes::AbstractVector{UInt8}, offset::Int64, + framesize::Int64, leaf::N5PhysicalLeaf; framelimit::Int=lastindex(bytes)) + first, last = _n5framerange(bytes, offset, framesize, framelimit) + reader = TH.Reader(bytes, first, last) + header = TH.decode(reader, MD.PageHeader) + headerlength = TH.consumed(reader) + payloadfirst = _n5checkedadd(first, headerlength, "page payload start") + compressedsize = Int(header.compressed_page_size) + compressedsize >= 0 || throw(ArgumentError( + "N5 compressed page size is negative")) + uncompressedsize = Int(header.uncompressed_page_size) + uncompressedsize >= 0 || throw(ArgumentError( + "N5 uncompressed page size is negative")) + payloadlast = _n5rangelast(payloadfirst, compressedsize, last, + "page payload") + payloadlast == last || throw(ArgumentError( + "N5 page compressed size differs from frame")) + uncompressedsize == compressedsize || + throw(ArgumentError("N5 focused decoder requires uncompressed pages")) + if header.type_ == MD.PageType.DATA_PAGE + data = header.data_page_header + data === nothing && throw(ArgumentError("N5 V1 page header is absent")) + data.encoding == MD.Encoding.PLAIN || throw(ArgumentError( + "N5 focused decoder requires PLAIN values")) + data.repetition_level_encoding == MD.Encoding.RLE || throw(ArgumentError( + "N5 focused decoder requires RLE repetition levels")) + data.definition_level_encoding == MD.Encoding.RLE || throw(ArgumentError( + "N5 focused decoder requires RLE definition levels")) + entries = Int(data.num_values) + 0 <= entries <= N5_MAX_DECODE_VALUES || throw(ArgumentError( + "N5 V1 value count exceeds the focused decoder limit")) + cursor = payloadfirst + repetition, cursor = n5decodehybrid(bytes, cursor, payloadlast, + entries, leaf.max_repetition; length_prefix=true) + definition, cursor = n5decodehybrid(bytes, cursor, payloadlast, + entries, leaf.max_definition; length_prefix=true) + dense = Base.count(==(UInt64(leaf.max_definition)), definition) + values, cursor = _n5plaindecode(bytes, cursor, payloadlast, dense, leaf) + cursor == _n5checkedadd(payloadlast, 1, "V1 payload end") || + throw(ArgumentError( + "N5 V1 value payload has trailing bytes")) + return N5LeafStream(repetition, definition, values, + leaf.max_repetition, leaf.max_definition), :v1 + elseif header.type_ == MD.PageType.DATA_PAGE_V2 + data = header.data_page_header_v2 + data === nothing && throw(ArgumentError("N5 V2 page header is absent")) + data.encoding == MD.Encoding.PLAIN || throw(ArgumentError( + "N5 focused decoder requires PLAIN values")) + data.is_compressed in (nothing, false) || throw(ArgumentError( + "N5 focused decoder requires uncompressed V2 values")) + entries = Int(data.num_values) + 0 <= entries <= N5_MAX_DECODE_VALUES || throw(ArgumentError( + "N5 V2 value count exceeds the focused decoder limit")) + rows = Int(data.num_rows) + 0 <= rows <= entries || throw(ArgumentError( + "N5 V2 row count is outside the value count")) + nulls = Int(data.num_nulls) + 0 <= nulls <= entries || throw(ArgumentError( + "N5 V2 null count is outside the value count")) + repetitionlength = Int(data.repetition_levels_byte_length) + repetitionlength >= 0 || throw(ArgumentError( + "N5 V2 repetition section length is negative")) + definitionlength = Int(data.definition_levels_byte_length) + definitionlength >= 0 || throw(ArgumentError( + "N5 V2 definition section length is negative")) + levellength = _n5checkedadd(repetitionlength, definitionlength, + "V2 level section length") + levellength <= payloadlast - payloadfirst + 1 || throw(ArgumentError( + "N5 V2 level sections exceed payload")) + repetitionlast = _n5rangelast(payloadfirst, repetitionlength, + payloadlast, "V2 repetition section") + definitionfirst = _n5checkedadd(repetitionlast, 1, + "V2 definition section start") + definitionlast = _n5rangelast(definitionfirst, definitionlength, + payloadlast, "V2 definition section") + repetition, repetitioncursor = n5decodehybrid(bytes, payloadfirst, + repetitionlast, entries, leaf.max_repetition; length_prefix=false) + repetitioncursor == _n5checkedadd(repetitionlast, 1, + "V2 repetition section end") || throw(ArgumentError( + "N5 V2 repetition section has trailing bytes")) + definition, definitioncursor = n5decodehybrid(bytes, definitionfirst, + definitionlast, entries, leaf.max_definition; length_prefix=false) + definitioncursor == _n5checkedadd(definitionlast, 1, + "V2 definition section end") || throw(ArgumentError( + "N5 V2 definition section has trailing bytes")) + dense = Base.count(==(UInt64(leaf.max_definition)), definition) + valuesfirst = _n5checkedadd(definitionlast, 1, + "V2 value section start") + values, cursor = _n5plaindecode(bytes, valuesfirst, + payloadlast, dense, leaf) + cursor == _n5checkedadd(payloadlast, 1, "V2 payload end") || + throw(ArgumentError( + "N5 V2 value payload has trailing bytes")) + Base.count(iszero, repetition) == rows || throw(ArgumentError( + "N5 V2 row count differs from repetition levels")) + Base.count(!=(UInt64(leaf.max_definition)), definition) == + nulls || throw(ArgumentError( + "N5 V2 null count differs from definition levels")) + return N5LeafStream(repetition, definition, values, + leaf.max_repetition, leaf.max_definition), :v2 + end + throw(ArgumentError("N5 focused decoder found a non-data page")) +end + +function n5decodefile(bytes::AbstractVector{UInt8}) + metadata, footeroffset = n5decodefooter(bytes) + leaves = n5physicalleaves(metadata.schema) + if isempty(metadata.row_groups) + metadata.num_rows == 0 || throw(ArgumentError( + "N5 file without row groups has nonzero rows")) + streams = N5LeafStream[N5LeafStream(UInt64[], UInt64[], Any[], + leaf.max_repetition, leaf.max_definition) for leaf in leaves] + return N5DecodedFile(metadata, leaves, streams, Symbol[]) + end + length(metadata.row_groups) == 1 || throw(ArgumentError( + "N5 focused decoder needs one row group")) + group = only(metadata.row_groups) + length(group.columns) == length(leaves) || throw(ArgumentError( + "N5 row-group column count differs from schema")) + streams = N5LeafStream[] + pageversions = Symbol[] + for (chunk, leaf) in zip(group.columns, leaves) + column = chunk.meta_data + column === nothing && throw(ArgumentError("N5 column metadata is absent")) + column.codec == MD.CompressionCodec.UNCOMPRESSED || throw(ArgumentError( + "N5 focused decoder requires UNCOMPRESSED columns")) + column.path_in_schema == leaf.path || throw(ArgumentError( + "N5 column path differs from schema")) + stream, pageversion = _n5decodepage(bytes, column.data_page_offset, + column.total_compressed_size, leaf; framelimit=footeroffset) + Int64(length(stream.repetition)) == column.num_values || + throw(ArgumentError("N5 column value count differs")) + push!(streams, stream) + push!(pageversions, pageversion) + end + group.num_rows == metadata.num_rows || throw(ArgumentError( + "N5 row-group row count differs from footer")) + return N5DecodedFile(metadata, leaves, streams, pageversions) +end diff --git a/test/conformance/n5/oracle-gate.sh b/test/conformance/n5/oracle-gate.sh new file mode 100755 index 0000000..0fa774f --- /dev/null +++ b/test/conformance/n5/oracle-gate.sh @@ -0,0 +1,336 @@ +#!/bin/sh +set -eu + +usage() { + echo "usage: oracle-gate.sh --repo REPO --corpus CORPUS --output DIR" >&2 + exit 64 +} + +n5_stage= +fail() { + echo "oracle-gate.sh: $1" >&2 + if [ -n "$n5_stage" ] && [ -d "$n5_stage" ]; then + printf '%s\n' "$1" > "$n5_stage/failure.txt" + fi + exit 65 +} + +is_hex64() { + n5_hex=$1 + [ "${#n5_hex}" -eq 64 ] || return 1 + case "$n5_hex" in + *[!0-9a-f]*) return 1 ;; + esac + return 0 +} + +require_plain_tree() { + n5_tree=$1 + n5_label=$2 + [ -d "$n5_tree" ] || fail "$n5_label directory is absent" + n5_special=$(find "$n5_tree" ! -type d ! -type f -print -quit) || + fail "cannot inspect $n5_label file types" + [ -z "$n5_special" ] || fail "$n5_label contains a non-regular file" +} + +check_sha_manifest() { + n5_manifest=$1 + n5_base=$2 + n5_expected_count=$3 + n5_label=$4 + [ -f "$n5_manifest" ] || fail "$n5_label hash manifest is absent" + awk ' + NF != 2 || length($1) != 64 || $1 !~ /^[0-9a-f]+$/ || + $2 ~ /^\// || $2 ~ /(^|\/)\.\.?($|\/)/ || $2 ~ /\\/ { exit 1 } + { print $2 } + ' "$n5_manifest" > "$n5_stage/$n5_label.paths" || + fail "$n5_label hash manifest is malformed" + n5_count=$(wc -l < "$n5_stage/$n5_label.paths") + [ "$n5_count" -eq "$n5_expected_count" ] || + fail "$n5_label hash manifest count differs" + LC_ALL=C sort "$n5_stage/$n5_label.paths" \ + > "$n5_stage/$n5_label.paths.sorted" || + fail "cannot sort $n5_label hash paths" + cmp -s "$n5_stage/$n5_label.paths" "$n5_stage/$n5_label.paths.sorted" || + fail "$n5_label hash paths are not sorted" + LC_ALL=C sort -u "$n5_stage/$n5_label.paths" \ + > "$n5_stage/$n5_label.paths.unique" || + fail "cannot deduplicate $n5_label hash paths" + n5_unique=$(wc -l < "$n5_stage/$n5_label.paths.unique") + [ "$n5_unique" -eq "$n5_expected_count" ] || + fail "$n5_label hash paths are not unique" + (cd "$n5_base" && sha256sum --check --quiet --strict "$n5_manifest") || + fail "$n5_label file hash differs" +} + +compare_fixture_directory() { + n5_generated=$1 + n5_checked=$2 + n5_count=$3 + n5_label=$4 + require_plain_tree "$n5_generated" "$n5_label generated fixtures" + require_plain_tree "$n5_checked" "$n5_label checked fixtures" + n5_subdirectory=$(find "$n5_generated" -mindepth 1 -type d -print -quit) || + fail "cannot inspect generated $n5_label directories" + [ -z "$n5_subdirectory" ] || + fail "$n5_label generated fixture tree contains a subdirectory" + n5_subdirectory=$(find "$n5_checked" -mindepth 1 -type d -print -quit) || + fail "cannot inspect checked $n5_label directories" + [ -z "$n5_subdirectory" ] || + fail "$n5_label checked fixture tree contains a subdirectory" + find "$n5_generated" -mindepth 1 -maxdepth 1 -type f \ + -print > "$n5_stage/$n5_label.generated.raw" || + fail "cannot list generated $n5_label fixtures" + sed 's#^.*/##' "$n5_stage/$n5_label.generated.raw" \ + > "$n5_stage/$n5_label.generated.unsorted" || + fail "cannot normalize generated $n5_label fixture names" + LC_ALL=C sort "$n5_stage/$n5_label.generated.unsorted" \ + > "$n5_stage/$n5_label.generated" || + fail "cannot normalize generated $n5_label fixture names" + find "$n5_checked" -mindepth 1 -maxdepth 1 -type f \ + -print > "$n5_stage/$n5_label.checked.raw" || + fail "cannot list checked $n5_label fixtures" + sed 's#^.*/##' "$n5_stage/$n5_label.checked.raw" \ + > "$n5_stage/$n5_label.checked.unsorted" || + fail "cannot normalize checked $n5_label fixture names" + LC_ALL=C sort "$n5_stage/$n5_label.checked.unsorted" \ + > "$n5_stage/$n5_label.checked" || + fail "cannot normalize checked $n5_label fixture names" + n5_actual=$(wc -l < "$n5_stage/$n5_label.generated") + [ "$n5_actual" -eq "$n5_count" ] || + fail "$n5_label generated fixture count differs" + cmp -s "$n5_stage/$n5_label.generated" "$n5_stage/$n5_label.checked" || + fail "$n5_label generated fixture names differ" + while IFS= read -r n5_name; do + cmp -s "$n5_generated/$n5_name" "$n5_checked/$n5_name" || + fail "$n5_label generated fixture differs: $n5_name" + done < "$n5_stage/$n5_label.generated" +} + +n5_repo= +n5_corpus= +n5_output= +while [ "$#" -gt 0 ]; do + case "$1" in + --repo) + [ "$#" -ge 2 ] || usage + n5_repo=$2 + shift 2 + ;; + --corpus) + [ "$#" -ge 2 ] || usage + n5_corpus=$2 + shift 2 + ;; + --output) + [ "$#" -ge 2 ] || usage + n5_output=$2 + shift 2 + ;; + *) usage ;; + esac +done +[ -n "$n5_repo" ] && [ -n "$n5_corpus" ] && [ -n "$n5_output" ] || usage +[ -d "$n5_repo/test/conformance/n5" ] || fail "N5 repository root is absent" +[ -d "$n5_corpus/.git" ] || fail "parquet-testing checkout is absent" +n5_parent=$(dirname "$n5_output") +[ -d "$n5_parent" ] || fail "output parent is absent" +if [ -e "$n5_output" ]; then + [ -d "$n5_output" ] || fail "output path is not a directory" + n5_output_entry=$(find "$n5_output" -mindepth 1 -print -quit) || + fail "cannot inspect output directory" + [ -z "$n5_output_entry" ] || + fail "output directory must be empty" +fi +n5_stage=$(mktemp -d "$n5_output.stage.XXXXXX") || fail "cannot create output stage" +[ "${N5_READONLY_CANARY_VERIFIED:-}" = 1 ] || + fail "read-only repository canary was not verified" +printf '%s\n' '{"schema_version":1,"repository_mount":"read-only"}' \ + > "$n5_stage/read-only-canary.json" || + fail "cannot record read-only repository canary" + +finish() { + n5_status=$? + trap - EXIT HUP INT TERM + set +e + if [ -d "$n5_stage" ]; then + if [ "$n5_status" -eq 0 ]; then + if ! printf '%s\n' '{"schema_version":1,"status":"ok"}' \ + > "$n5_stage/status.json"; then + n5_status=73 + elif ! find "$n5_stage" -type f ! -name evidence.sha256 \ + ! -name evidence.raw ! -name evidence.unsorted \ + ! -name evidence.files -print > "$n5_stage/evidence.raw"; then + n5_status=73 + elif ! sed "s#^$n5_stage/##" "$n5_stage/evidence.raw" \ + > "$n5_stage/evidence.unsorted"; then + n5_status=73 + elif ! LC_ALL=C sort "$n5_stage/evidence.unsorted" \ + > "$n5_stage/evidence.files"; then + n5_status=73 + elif ! (cd "$n5_stage" && while IFS= read -r n5_file; do + sha256sum "$n5_file" || exit 1 + done < evidence.files > evidence.sha256); then + n5_status=73 + elif ! rm "$n5_stage/evidence.raw" "$n5_stage/evidence.unsorted" \ + "$n5_stage/evidence.files"; then + n5_status=73 + fi + if [ "$n5_status" -ne 0 ]; then + echo "oracle-gate.sh: cannot finalize evidence" >&2 + rm -f "$n5_stage/status.json" "$n5_stage/evidence.sha256" + printf '%s\n' "cannot finalize evidence" \ + > "$n5_stage/failure.txt" + fi + elif [ ! -f "$n5_stage/failure.txt" ]; then + printf '%s\n' "oracle gate exited with status $n5_status" \ + > "$n5_stage/failure.txt" + fi + if [ -d "$n5_output" ]; then + rmdir "$n5_output" || { + echo "oracle-gate.sh: cannot commit evidence over output" >&2 + exit 73 + } + fi + mv "$n5_stage" "$n5_output" || { + echo "oracle-gate.sh: cannot commit evidence" >&2 + exit 73 + } + fi + exit "$n5_status" +} +trap finish EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +n5_root="$n5_repo/test/conformance/n5" +n5_fixtures="$n5_root/julia-fixtures" +require_plain_tree "$n5_fixtures" "Julia fixture corpus" + +check_sha_manifest "$n5_root/external-files.sha256" "$n5_repo" 42 external +{ + find "$n5_root/golden" "$n5_root/expected" -type f -print + printf '%s\n' "$n5_root/manifest.toml" +} > "$n5_stage/external.actual.raw" || fail "cannot list external checked files" +sed "s#^$n5_repo/##" "$n5_stage/external.actual.raw" \ + > "$n5_stage/external.actual.unsorted" || + fail "cannot normalize external checked files" +LC_ALL=C sort "$n5_stage/external.actual.unsorted" \ + > "$n5_stage/external.actual" || + fail "cannot normalize external checked files" +cmp -s "$n5_stage/external.paths" "$n5_stage/external.actual" || + fail "external checked file set differs" + +n5_corpus_commit=$(git -C "$n5_corpus" rev-parse HEAD) || + fail "cannot read parquet-testing revision" +[ "$n5_corpus_commit" = "09f3cdbde45302f0f0c689c950e465e98a9df960" ] || + fail "parquet-testing revision differs" +check_sha_manifest "$n5_root/corpus-files.sha256" "$n5_corpus" 14 corpus + +check_sha_manifest "$n5_root/julia-fixtures.files.sha256" "$n5_root" 1 \ + julia-fixture-manifest +check_sha_manifest "$n5_fixtures/files.sha256" "$n5_fixtures" 353 julia-fixtures +find "$n5_fixtures" -type f -print > "$n5_stage/julia-fixtures.actual.raw" || + fail "cannot list Julia fixture corpus" +sed "s#^$n5_fixtures/##" "$n5_stage/julia-fixtures.actual.raw" \ + > "$n5_stage/julia-fixtures.relative" || + fail "cannot normalize Julia fixture corpus" +awk '$0 != "files.sha256" { print }' "$n5_stage/julia-fixtures.relative" \ + > "$n5_stage/julia-fixtures.unsorted" || + fail "cannot normalize Julia fixture corpus" +LC_ALL=C sort "$n5_stage/julia-fixtures.unsorted" \ + > "$n5_stage/julia-fixtures.actual" || + fail "cannot normalize Julia fixture corpus" +cmp -s "$n5_stage/julia-fixtures.paths" "$n5_stage/julia-fixtures.actual" || + fail "Julia fixture file set differs" + +mkdir "$n5_stage/java-generated" "$n5_stage/rust-generated" \ + "$n5_stage/rust-owned-checked" "$n5_stage/rust-neighbor-generated" || + fail "cannot create generator stages" +if ! /opt/bootstrap/parquet-java/run.sh --offline generate \ + --output "$n5_stage/java-generated" \ + --evidence "$n5_stage/java-generated.jsonl" \ + > "$n5_stage/java-generate.log" 2>&1; then + fail "Parquet Java fixture generation failed" +fi +compare_fixture_directory "$n5_stage/java-generated" \ + "$n5_root/golden/parquet-java" 30 parquet-java +cmp -s "$n5_stage/java-generated.jsonl" \ + "$n5_root/expected/parquet-java.jsonl" || + fail "Parquet Java generated evidence differs" + +if ! parquet-jl-n5-arrow-rs-oracle generate \ + --output "$n5_stage/rust-generated" \ + --evidence "$n5_stage/rust-generated.json" \ + > "$n5_stage/rust-generate.log" 2>&1; then + fail "Arrow Rust fixture generation failed" +fi +find "$n5_root/golden/arrow-rs" -mindepth 1 -maxdepth 1 -type f \ + -name '*.parquet' ! -name '*near-neighbor*' -print \ + > "$n5_stage/rust-owned.checked.raw" || + fail "cannot list checked Arrow Rust owned fixtures" +while IFS= read -r n5_file; do + cp "$n5_file" "$n5_stage/rust-owned-checked/" || + fail "cannot stage checked Arrow Rust owned fixture" +done < "$n5_stage/rust-owned.checked.raw" +compare_fixture_directory "$n5_stage/rust-generated" \ + "$n5_stage/rust-owned-checked" 6 arrow-rs-owned +cmp -s "$n5_stage/rust-generated.json" "$n5_root/expected/arrow-rs.json" || + fail "Arrow Rust generated evidence differs" + +if ! parquet-jl-n5-arrow-rs-oracle diagnose-rule3-near-neighbor \ + --output "$n5_stage/rust-neighbor-generated" \ + --evidence "$n5_stage/rust-neighbor-generated.json" \ + > "$n5_stage/rust-neighbor-generate.log" 2>&1; then + fail "Arrow Rust near-neighbor generation failed" +fi +find "$n5_root/golden/arrow-rs" -mindepth 1 -maxdepth 1 -type f \ + -name '*near-neighbor*' -print > "$n5_stage/rust-neighbor.checked.raw" || + fail "cannot list checked Arrow Rust near-neighbor fixtures" +mkdir "$n5_stage/rust-neighbor-checked" || + fail "cannot create Arrow Rust near-neighbor comparison stage" +while IFS= read -r n5_file; do + cp "$n5_file" "$n5_stage/rust-neighbor-checked/" || + fail "cannot stage checked Arrow Rust near-neighbor fixture" +done < "$n5_stage/rust-neighbor.checked.raw" +compare_fixture_directory "$n5_stage/rust-neighbor-generated" \ + "$n5_stage/rust-neighbor-checked" 2 arrow-rs-neighbor +cmp -s "$n5_stage/rust-neighbor-generated.json" \ + "$n5_root/expected/arrow-rs-rule3-near-neighbor.json" || + fail "Arrow Rust near-neighbor evidence differs" + +if ! /opt/bootstrap/parquet-java/run.sh --offline audit \ + --input "$n5_fixtures" --evidence "$n5_stage/java-audit.jsonl" \ + > "$n5_stage/java-audit.log" 2>&1; then + fail "Parquet Java Julia-fixture audit failed" +fi +if ! parquet-jl-n5-arrow-rs-oracle audit \ + --input "$n5_fixtures" --evidence "$n5_stage/rust-audit.json" \ + > "$n5_stage/rust-audit.log" 2>&1; then + fail "Arrow Rust Julia-fixture audit failed" +fi +if ! perl "$n5_root/compare-evidence.pl" \ + --manifest "$n5_fixtures/fixture-manifest.tsv" \ + --unsupported "$n5_root/oracle-unsupported.tsv" \ + --java "$n5_stage/java-audit.jsonl" \ + --rust "$n5_stage/rust-audit.json" \ + --output "$n5_stage/summary.json" \ + > "$n5_stage/compare.log" 2>&1; then + fail "Julia fixture evidence comparison failed" +fi + +cp "$n5_root/external-files.sha256" "$n5_stage/" || + fail "cannot preserve external file manifest" +cp "$n5_root/corpus-files.sha256" "$n5_stage/" || + fail "cannot preserve corpus file manifest" +cp "$n5_root/julia-fixtures.files.sha256" "$n5_stage/" || + fail "cannot preserve Julia fixture manifest digest" +cp "$n5_fixtures/files.sha256" "$n5_stage/julia-files.sha256" || + fail "cannot preserve Julia fixture hashes" +cp "$n5_fixtures/fixture-manifest.tsv" "$n5_stage/" || + fail "cannot preserve Julia fixture mapping manifest" +cp "$n5_root/oracle-unsupported.tsv" "$n5_stage/" || + fail "cannot preserve oracle unsupported allowlist" + +exit 0 diff --git a/test/conformance/n5/oracle-unsupported.tsv b/test/conformance/n5/oracle-unsupported.tsv new file mode 100644 index 0000000..e52d429 --- /dev/null +++ b/test/conformance/n5/oracle-unsupported.tsv @@ -0,0 +1,71 @@ +oracle file error_class error_message +arrow-rs julia/model/schema-provenance.v1.parquet error Parquet error: Empty struct has fields +arrow-rs julia/model/schema-provenance.v2.parquet error Parquet error: Empty struct has fields +arrow-rs reference/model/schema-provenance.v1.parquet error Parquet error: Empty struct has fields +arrow-rs reference/model/schema-provenance.v2.parquet error Parquet error: Empty struct has fields +parquet-java julia/model/schema-provenance.v1.parquet java.lang.NullPointerException +parquet-java julia/model/schema-provenance.v2.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0002-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0004-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0005-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0006-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0007-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0008-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0009-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0010-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0011-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0014-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0015-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0018-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0028-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0034-v2-brotli.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0034-v2-gzip.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0034-v2-lz4_raw.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0034-v2-snappy.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0034-v2-uncompressed.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0034-v2-zstd.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0038-v2-brotli.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0038-v2-gzip.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0038-v2-lz4_raw.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0038-v2-snappy.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0038-v2-uncompressed.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0038-v2-zstd.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0048-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0058-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0062-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0096-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0106-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0107-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0117-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0122-v2-brotli.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0122-v2-gzip.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0122-v2-lz4_raw.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0122-v2-snappy.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0122-v2-uncompressed.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0122-v2-zstd.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0123-v1-brotli.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0123-v1-gzip.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0123-v1-lz4_raw.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0123-v1-snappy.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0123-v1-uncompressed.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0123-v1-zstd.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0144-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0145-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0148-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0178-v1-brotli.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0178-v1-gzip.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0178-v1-lz4_raw.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0178-v1-snappy.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0178-v1-uncompressed.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0178-v1-zstd.parquet java.lang.NullPointerException +parquet-java julia/property/generated-0191-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0202-v2-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0249-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java julia/property/generated-0251-v1-brotli.parquet java.lang.ClassNotFoundException org.apache.hadoop.io.compress.BrotliCodec +parquet-java reference/model/schema-provenance.v1.parquet java.lang.NullPointerException +parquet-java reference/model/schema-provenance.v2.parquet java.lang.NullPointerException +parquet-java reference/property/generated-0034-v2.parquet java.lang.NullPointerException +parquet-java reference/property/generated-0038-v2.parquet java.lang.NullPointerException +parquet-java reference/property/generated-0122-v2.parquet java.lang.NullPointerException +parquet-java reference/property/generated-0123-v1.parquet java.lang.NullPointerException +parquet-java reference/property/generated-0178-v1.parquet java.lang.NullPointerException diff --git a/test/conformance/n5/oracles/.dockerignore b/test/conformance/n5/oracles/.dockerignore new file mode 100644 index 0000000..7cf8be6 --- /dev/null +++ b/test/conformance/n5/oracles/.dockerignore @@ -0,0 +1,3 @@ +**/target +**/target/** +**/.DS_Store diff --git a/test/conformance/n5/oracles/Dockerfile b/test/conformance/n5/oracles/Dockerfile new file mode 100644 index 0000000..012b322 --- /dev/null +++ b/test/conformance/n5/oracles/Dockerfile @@ -0,0 +1,191 @@ +ARG BASE_IMAGE=eclipse-temurin:11.0.28_6-jdk@sha256:ab2527b3c9b7c15bc88f60dec19b2aa39939a6e0045fb8f538eeecbd7af59c69 +ARG TARGETPLATFORM=linux/amd64 +ARG SOURCE_DATE_EPOCH=1787356800 +FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE} + +ARG BASE_IMAGE +ARG SOURCE_DATE_EPOCH +ARG UBUNTU_SNAPSHOT=20260822T000000Z +ARG BUILD_ESSENTIAL_VERSION=12.10ubuntu1 +ARG CA_CERTIFICATES_VERSION=20260601~24.04.1 +ARG CMAKE_VERSION=3.28.3-1build7 +ARG GIT_VERSION=1:2.43.0-1ubuntu7.3 +ARG PKG_CONFIG_VERSION=1.8.1-2build1 +ARG XZ_UTILS_VERSION=5.6.1+really5.4.5-1ubuntu0.3 +ARG MAVEN_VERSION=3.9.8 +ARG MAVEN_SHA512=7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2 +ARG RUST_VERSION=1.96.1 +ARG RUST_DIST_DATE=2026-06-30 +ARG RUST_CHANNEL_SHA256=87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975 +ARG RUST_TARBALL_SHA256=d29ccb1559a177c4e72291f6e5f629de7fe8885e7521ca47802627544b121e95 +ARG PERL_PACKAGE_VERSION=5.38.2-3.2ubuntu0.3 +ARG PERL_VERSION=v5.38.2 +ARG JSON_PP_VERSION=4.16 + +ENV CARGO_HOME=/opt/n5/cargo \ + CARGO_NET_OFFLINE=true \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + MAVEN_OPTS=-Dmaven.repo.local=/opt/n5/maven/repository \ + SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} \ + TZ=UTC + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN set -eux; \ + snapshot="https://snapshot.ubuntu.com/ubuntu/${UBUNTU_SNAPSHOT}"; \ + sed -i \ + -e "s#http://archive.ubuntu.com/ubuntu/#${snapshot}/#" \ + -e "s#http://security.ubuntu.com/ubuntu/#${snapshot}/#" \ + /etc/apt/sources.list.d/ubuntu.sources; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ + "build-essential=${BUILD_ESSENTIAL_VERSION}" \ + "ca-certificates=${CA_CERTIFICATES_VERSION}" \ + "cmake=${CMAKE_VERSION}" \ + "git=${GIT_VERSION}" \ + "perl=${PERL_PACKAGE_VERSION}" \ + "pkg-config=${PKG_CONFIG_VERSION}" \ + "xz-utils=${XZ_UTILS_VERSION}"; \ + test "$(dpkg-query -W -f='${Version}' build-essential)" = "$BUILD_ESSENTIAL_VERSION"; \ + test "$(dpkg-query -W -f='${Version}' ca-certificates)" = "$CA_CERTIFICATES_VERSION"; \ + test "$(dpkg-query -W -f='${Version}' cmake)" = "$CMAKE_VERSION"; \ + test "$(dpkg-query -W -f='${Version}' git)" = "$GIT_VERSION"; \ + test "$(dpkg-query -W -f='${Version}' perl)" = "$PERL_PACKAGE_VERSION"; \ + test "$(dpkg-query -W -f='${Version}' pkg-config)" = "$PKG_CONFIG_VERSION"; \ + test "$(dpkg-query -W -f='${Version}' xz-utils)" = "$XZ_UTILS_VERSION"; \ + rm -rf /var/cache/* /var/lib/apt/lists/*; \ + find /var/log -type f -delete + +RUN set -eux; \ + archive="apache-maven-${MAVEN_VERSION}-bin.tar.gz"; \ + url="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${archive}"; \ + curl --fail --location --silent --show-error "$url" --output "/tmp/${archive}"; \ + printf '%s %s\n' "$MAVEN_SHA512" "/tmp/${archive}" | sha512sum --check --strict; \ + tar --extract --gzip --file "/tmp/${archive}" --directory /opt; \ + ln --symbolic "/opt/apache-maven-${MAVEN_VERSION}/bin/mvn" /usr/local/bin/mvn; \ + rm "/tmp/${archive}"; \ + test "$(mvn --version | awk 'NR == 1 { print $3 }')" = "$MAVEN_VERSION" + +RUN set -eux; \ + channel="channel-rust-${RUST_VERSION}.toml"; \ + channel_url="https://static.rust-lang.org/dist/${channel}"; \ + archive="rust-${RUST_VERSION}-x86_64-unknown-linux-gnu.tar.xz"; \ + archive_url="https://static.rust-lang.org/dist/${RUST_DIST_DATE}/${archive}"; \ + curl --fail --location --silent --show-error "$channel_url" --output "/tmp/${channel}"; \ + printf '%s %s\n' "$RUST_CHANNEL_SHA256" "/tmp/${channel}" | sha256sum --check --strict; \ + curl --fail --location --silent --show-error "$archive_url" --output "/tmp/${archive}"; \ + printf '%s %s\n' "$RUST_TARBALL_SHA256" "/tmp/${archive}" | sha256sum --check --strict; \ + tar --extract --xz --file "/tmp/${archive}" --directory /tmp; \ + "/tmp/rust-${RUST_VERSION}-x86_64-unknown-linux-gnu/install.sh" \ + --prefix=/usr/local \ + --disable-ldconfig \ + --components=rustc,cargo,rust-std-x86_64-unknown-linux-gnu,rustfmt-preview; \ + install --directory /opt/n5/pins; \ + install --mode=0444 "/tmp/${channel}" /opt/n5/pins/; \ + rm -rf "/tmp/${archive}" "/tmp/${channel}" \ + "/tmp/rust-${RUST_VERSION}-x86_64-unknown-linux-gnu"; \ + test "$(rustc --version)" = "rustc ${RUST_VERSION} (31fca3adb 2026-06-26)"; \ + test "$(cargo --version | sed 's/ (.*//')" = "cargo ${RUST_VERSION}" + +COPY parquet-java /opt/bootstrap/parquet-java +COPY arrow-rs /opt/bootstrap/arrow-rs +COPY image/validate-image.sh /usr/local/bin/validate-n5-oracle-image + +RUN set -eux; \ + chmod 0555 /usr/local/bin/validate-n5-oracle-image; \ + install --directory /opt/n5/maven/repository /opt/n5/manifests; \ + cd /opt/bootstrap/parquet-java; \ + mvn --batch-mode --no-transfer-progress clean test package; \ + mvn --batch-mode --no-transfer-progress --offline clean test package; \ + mvn --batch-mode --no-transfer-progress --offline dependency:tree \ + -DoutputFile=/opt/n5/manifests/maven-dependency-tree.txt \ + -DappendOutput=false; \ + rm -rf /opt/bootstrap/parquet-java/target; \ + find /opt/n5/maven/repository -type f \ + \( -name _remote.repositories -o -name resolver-status.properties \ + -o -name '*.lastUpdated' \) -delete; \ + rm -rf /tmp/* + +RUN set -eux; \ + export CARGO_NET_OFFLINE=false; \ + cd /opt/bootstrap/arrow-rs; \ + cargo fetch --locked; \ + cargo vendor --locked --versioned-dirs /opt/n5/vendor > /opt/n5/cargo/config.toml; \ + export CARGO_NET_OFFLINE=true; \ + cargo test --offline --locked; \ + cargo build --release --offline --locked; \ + cargo tree --locked > /opt/n5/manifests/cargo-dependency-tree.txt; \ + install --mode=0555 target/release/parquet-jl-n5-arrow-rs-oracle \ + /usr/local/bin/parquet-jl-n5-arrow-rs-oracle; \ + rm -rf /opt/bootstrap/arrow-rs/target; \ + find /opt/n5/cargo -mindepth 1 -maxdepth 1 ! -name config.toml \ + -exec rm -rf -- '{}' +; \ + rm -rf /tmp/* + +RUN set -eux; \ + test "$(dpkg-query -W -f='${Version}' perl)" = "$PERL_PACKAGE_VERSION"; \ + test "$(perl -e 'print $^V')" = "$PERL_VERSION"; \ + test "$(perl -MJSON::PP -e 'print $JSON::PP::VERSION')" = "$JSON_PP_VERSION"; \ + printf '%s\n' \ + "base_image=${BASE_IMAGE}" \ + "source_date_epoch=${SOURCE_DATE_EPOCH}" \ + "ubuntu_snapshot=${UBUNTU_SNAPSHOT}" \ + "ubuntu_build_essential=${BUILD_ESSENTIAL_VERSION}" \ + "ubuntu_ca_certificates=${CA_CERTIFICATES_VERSION}" \ + "ubuntu_cmake=${CMAKE_VERSION}" \ + "ubuntu_git=${GIT_VERSION}" \ + "ubuntu_pkg_config=${PKG_CONFIG_VERSION}" \ + "ubuntu_xz_utils=${XZ_UTILS_VERSION}" \ + "maven_version=${MAVEN_VERSION}" \ + "maven_archive_sha512=${MAVEN_SHA512}" \ + "rust_version=${RUST_VERSION}" \ + "rust_channel_manifest_sha256=${RUST_CHANNEL_SHA256}" \ + "rust_tarball_sha256=${RUST_TARBALL_SHA256}" \ + "perl_package_version=${PERL_PACKAGE_VERSION}" \ + "perl_version=${PERL_VERSION}" \ + "json_pp_version=${JSON_PP_VERSION}" \ + 'parquet_java_version=1.17.1' \ + 'parquet_java_commit=78a8d3230eb4769db93de5f2f2e18363c04cae81' \ + 'arrow_rs_version=59.2.0' \ + 'arrow_rs_commit=782e5a685501a9db6cc8e9a3b7cbff894940c47a' \ + > /opt/n5/manifests/toolchains.txt; \ + test -z "$(find /opt/n5/maven/repository /opt/n5/vendor /opt/n5/cargo \ + /opt/n5/pins /opt/bootstrap ! -type d ! -type f -print -quit)"; \ + cd /opt/n5/maven/repository; \ + find . -type f -print0 > /tmp/n5-maven-files; \ + LC_ALL=C sort -z /tmp/n5-maven-files -o /tmp/n5-maven-files; \ + xargs -0 sha256sum < /tmp/n5-maven-files \ + > /opt/n5/manifests/maven-artifacts.sha256; \ + cd /; \ + find opt/n5/vendor opt/n5/cargo opt/n5/pins -type f -print0 \ + > /tmp/n5-cargo-files; \ + printf '%s\000' usr/local/bin/parquet-jl-n5-arrow-rs-oracle \ + >> /tmp/n5-cargo-files; \ + LC_ALL=C sort -z /tmp/n5-cargo-files -o /tmp/n5-cargo-files; \ + xargs -0 sha256sum < /tmp/n5-cargo-files \ + > /opt/n5/manifests/cargo-vendor.sha256; \ + find opt/bootstrap -type f -print0 > /tmp/n5-source-files; \ + printf '%s\000' opt/n5/manifests/toolchains.txt \ + usr/local/bin/validate-n5-oracle-image >> /tmp/n5-source-files; \ + LC_ALL=C sort -z /tmp/n5-source-files -o /tmp/n5-source-files; \ + xargs -0 sha256sum < /tmp/n5-source-files \ + > /opt/n5/manifests/harness-source.sha256; \ + rm /tmp/n5-maven-files /tmp/n5-cargo-files /tmp/n5-source-files; \ + test -s /opt/n5/manifests/maven-artifacts.sha256; \ + test -s /opt/n5/manifests/cargo-vendor.sha256; \ + test -s /opt/n5/manifests/harness-source.sha256; \ + cd /opt/n5/maven/repository; \ + sha256sum --check --quiet --strict /opt/n5/manifests/maven-artifacts.sha256; \ + cd /; \ + sha256sum --check --quiet --strict /opt/n5/manifests/cargo-vendor.sha256; \ + sha256sum --check --quiet --strict /opt/n5/manifests/harness-source.sha256; \ + chmod 0444 /opt/n5/cargo/config.toml /opt/n5/manifests/*; \ + find /opt/n5 /opt/bootstrap -exec touch --no-dereference \ + --date="@${SOURCE_DATE_EPOCH}" '{}' +; \ + touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" \ + /usr/local/bin/parquet-jl-n5-arrow-rs-oracle \ + /usr/local/bin/validate-n5-oracle-image + +WORKDIR /work +ENTRYPOINT ["/usr/local/bin/validate-n5-oracle-image"] diff --git a/test/conformance/n5/oracles/README.md b/test/conformance/n5/oracles/README.md new file mode 100644 index 0000000..051bb9e --- /dev/null +++ b/test/conformance/n5/oracles/README.md @@ -0,0 +1,49 @@ +# N5 locked oracle image + +The Dockerfile builds the test-only Linux/amd64 image that contains Parquet Java +1.17.1, Arrow Rust 59.2.0, Temurin 11.0.28+6, Maven 3.9.8, and Rust 1.96.1. +The image also binds Perl 5.38.2 and JSON::PP 4.16 for the evidence comparator. +Every downloaded archive has a checked digest. Build packages come from the +immutable Ubuntu `20260822T000000Z` snapshot at exact versions. The image contains +the complete Maven repository and Cargo vendor tree. Closed-world manifests also +cover the Cargo source replacement, installed Rust oracle, saved Rust channel, +dependency trees, exact toolchain record, validator, and harness source. Its +validator checks all content and runs both harnesses without network access. +The fixed `SOURCE_DATE_EPOCH=1787356800` controls image metadata. The Docker +exporter rewrites retained file times to that epoch. Volatile package-manager +logs and Maven resolver records are absent. + +Build and validate a local image without publishing it: + +```sh +test/conformance/n5/bootstrap-oracles.sh --output /tmp/oracles.lock +``` + +The local command exits with status 2 after successful validation. It does not +write a binding lock because a local image ID is not a registry `RepoDigest`. +Publishing is a separate external action. The bootstrap always makes a no-cache +Linux/amd64 build and checks it with `--network none` before any push. If the +output lock already exists, the clean image ID must equal its locked image ID. +After publication is authorized, pass the one permitted repository explicitly: + +```sh +test/conformance/n5/bootstrap-oracles.sh \ + --output test/conformance/n5/oracles.lock \ + --publish ghcr.io/juliaio/parquet-jl-n5-oracles +``` + +Publication uses a staging tag derived from the validated local image ID. The +bootstrap pulls the resulting digest, verifies that it has the same image ID and +`RepoDigest`, and repeats the offline validation before it writes a lock. + +The binding runner requires the exact 22-field lock schema, fixed public +repository, Linux/amd64 platform, toolchain pins, upstream revisions, dependency +trees, content manifests, corpus commit, and fixture hash. It may pull only that +exact public digest. It verifies the digest and content bindings before it starts +the full gate with `--network none`: + +```sh +test/conformance/n5/run-oracles.sh \ + --lock test/conformance/n5/oracles.lock \ + --network none +``` diff --git a/test/conformance/n5/oracles/arrow-rs/.gitignore b/test/conformance/n5/oracles/arrow-rs/.gitignore new file mode 100644 index 0000000..92055b7 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/.gitignore @@ -0,0 +1,2 @@ +/target/ +*.tmp diff --git a/test/conformance/n5/oracles/arrow-rs/Cargo.lock b/test/conformance/n5/oracles/arrow-rs/Cargo.lock new file mode 100644 index 0000000..346113f --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/Cargo.lock @@ -0,0 +1,1082 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-json" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" + +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parquet" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=782e5a685501a9db6cc8e9a3b7cbff894940c47a#782e5a685501a9db6cc8e9a3b7cbff894940c47a" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "seq-macro", + "simdutf8", + "snap", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet-jl-n5-arrow-rs-oracle" +version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-json", + "arrow-schema", + "parquet", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/test/conformance/n5/oracles/arrow-rs/Cargo.toml b/test/conformance/n5/oracles/arrow-rs/Cargo.toml new file mode 100644 index 0000000..a587074 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "parquet-jl-n5-arrow-rs-oracle" +version = "0.1.0" +edition = "2024" +rust-version = "1.96.1" +publish = false +license = "MIT" +description = "Offline Arrow Rust conformance oracle for Parquet.jl N5" + +[dependencies] +arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +arrow-json = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a", default-features = true } +serde = { version = "=1.0.228", features = ["derive"] } +serde_json = "=1.0.145" +sha2 = "=0.10.9" + +[profile.release] +codegen-units = 1 +lto = "thin" +strip = "debuginfo" diff --git a/test/conformance/n5/oracles/arrow-rs/README.md b/test/conformance/n5/oracles/arrow-rs/README.md new file mode 100644 index 0000000..4f0d011 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/README.md @@ -0,0 +1,141 @@ +# N5 Arrow Rust oracle + +This test-only executable owns the Arrow Rust N5 producer cases and provides a +general low-level reader for Julia, Java, and Rust Parquet fixtures. It is not a +Parquet.jl dependency. + +## Pins + +- Arrow Rust tag: `59.2.0` +- Arrow Rust commit: `782e5a685501a9db6cc8e9a3b7cbff894940c47a` +- Rust toolchain: `1.96.1` +- Rust compiler identity: `rustc 1.96.1 (31fca3adb 2026-06-26)` +- Cargo identity: `cargo 1.96.1 (356927216 2026-06-26)` +- Rust channel manifest SHA-256: + `87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975` + +`Cargo.toml` pins all Arrow crates to the exact Git commit. `Cargo.lock` pins the +complete crate graph. `rust-toolchain.toml` selects the exact compiler. The +common N5 OCI bootstrap vendors this lock graph and records the vendor hashes in +`oracles.lock`. + +The binding CI gate runs only inside the locked Linux/amd64 OCI image. Local +execution on another platform is useful validation, but is not a substitute for +that gate. + +## Offline build and test + +The checked scripts always use `--offline --locked` and set +`CARGO_NET_OFFLINE=true`: + +```sh +scripts/build.sh +scripts/test.sh +``` + +A networked maintainer bootstrap must fetch the exact lock graph before these +commands run. The repository-level OCI bootstrap owns that operation. The gate +must not change the lock file or access the network. + +## Generate owned fixtures + +```sh +scripts/run.sh generate \ + --output /work/golden/arrow-rs \ + --evidence /work/evidence/arrow-rs-generate.json +``` + +Generation writes six files: + +- `arrow-rs-duplicate-keys_v1.parquet` +- `arrow-rs-duplicate-keys_v2.parquet` +- `arrow-rs-optional-key-present_v1.parquet` +- `arrow-rs-optional-key-present_v2.parquet` +- `arrow-rs-list-rule3_v1.parquet` +- `arrow-rs-list-rule3_v2.parquet` + +The writer uses explicit repetition and definition levels. It disables +dictionary encoding, compression, and statistics. It fixes `created_by` and all +writer properties that affect these files. Generation immediately reads every +file through the independent low-level column and page APIs. It also uses the +Arrow `RecordBatch` reader. A mismatch exits with a nonzero status. + +The duplicate-key `RecordBatch` check reads the underlying `MapArray` entries. +It does not project the map through a Rust or JSON map. Thus duplicate keys and +their order remain observable. JSON lines are diagnostic only; they can contain +duplicate object member names. + +The owned LIST rule-3 fixture follows the Parquet 2.13 compatibility example. +Its repeated `array` group carries a LIST annotation and contains a repeated +`INT32` field. The low-level check requires the binding repetition, definition, +and dense-value streams. The `RecordBatch` check separately requires the exact +type `List>` and rows `null`, `[]`, `[[]]`, and +`[[1,2],[],[3]]`. + +## Inspect another fixture + +```sh +scripts/run.sh inspect \ + --input /work/julia/direct-list-of-map.parquet \ + --case-id julia-direct-list-of-map-v1 \ + --evidence /work/evidence/julia-direct-list-of-map-v1.arrow-rs.json +``` + +The machine-readable evidence contains: + +- the exact file SHA-256 and row-group count; +- the printed physical Parquet schema; +- every leaf path, physical type, and maximum level; +- per-row-group repetition, definition, and dense-value streams; +- every data-page version, encoding, value count, and derived row count; +- canonical Arrow schema evidence and ordered high-level rows when representable; and +- an explicit `unsupported` diagnostic when Arrow cannot represent a layout. + +Unsupported high-level Arrow semantics do not erase low-level evidence. The +command fails if the low-level reader cannot parse the physical file. Canonical +rows encode structs as ordered fields and maps as ordered key/value pairs. The +separate Arrow JSON lines are diagnostic. For example, Arrow can represent maps +with non-string keys even though its JSON writer cannot serialize them. + +## Unannotated LIST rule-3 near-neighbor + +This diagnostic removes the inner LIST annotation from the repeated one-child +group. It is not the binding Rule 3 case. Under pinned Arrow Rust 59.2.0, the +`RecordBatch` reader reports `List>` and rows `null`, `[]`, `[[]]`, +and `[[1,2],[],[3]]`. The checked evidence records this actual outcome without +using it as the specification authority. + +```sh +scripts/run.sh diagnose-rule3-near-neighbor \ + --output /work/diagnostics/rule3-near-neighbor \ + --evidence /work/evidence/arrow-rs-rule3-near-neighbor.json +``` + +## Compare with checked evidence + +The `verify` command accepts one checked `FileEvidence` JSON object or a complete +oracle report containing the matching case ID and file name. It compares all +file fields, including the hash, schema, streams, pages, and high-level outcome. + +```sh +scripts/run.sh verify \ + --input /work/julia/standard-map-v1.parquet \ + --expected /work/expected/standard-map-v1.arrow-rs.json \ + --case-id julia-standard-map-v1 \ + --evidence /work/evidence/standard-map-v1.verify.json +``` + +`generate` and `inspect` wrap file objects in a top-level oracle report. The +repository-level manifest task can retain that report or store each selected +`files[]` object as the input to `verify`. + +## Version evidence + +```sh +scripts/run.sh versions --evidence /work/evidence/arrow-rs-versions.json +``` + +Every report records the Arrow version, upstream commit, selected Rust +toolchain, and exact `rustc --version` and `cargo --version` results. Every +command fails when either tool is absent or differs from the pins above. Missing +inputs and unsupported commands also fail. The harness has no absence skip. diff --git a/test/conformance/n5/oracles/arrow-rs/UPSTREAM.toml b/test/conformance/n5/oracles/arrow-rs/UPSTREAM.toml new file mode 100644 index 0000000..7f27be8 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/UPSTREAM.toml @@ -0,0 +1,11 @@ +arrow_rs_tag = "59.2.0" +arrow_rs_commit = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +rust_toolchain = "1.96.1" +rust_channel_manifest_sha256 = "87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975" + +[crates] +arrow_array = "59.2.0" +arrow_cast = "59.2.0" +arrow_json = "59.2.0" +arrow_schema = "59.2.0" +parquet = "59.2.0" diff --git a/test/conformance/n5/oracles/arrow-rs/rust-toolchain.toml b/test/conformance/n5/oracles/arrow-rs/rust-toolchain.toml new file mode 100644 index 0000000..f40d3c1 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.96.1" +profile = "minimal" +components = ["rustfmt"] diff --git a/test/conformance/n5/oracles/arrow-rs/scripts/build.sh b/test/conformance/n5/oracles/arrow-rs/scripts/build.sh new file mode 100755 index 0000000..214d916 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/scripts/build.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +export CARGO_NET_OFFLINE=true +cd "$oracle_dir" +exec cargo build --manifest-path Cargo.toml --release --offline --locked diff --git a/test/conformance/n5/oracles/arrow-rs/scripts/run.sh b/test/conformance/n5/oracles/arrow-rs/scripts/run.sh new file mode 100755 index 0000000..7c2e228 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/scripts/run.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +export CARGO_NET_OFFLINE=true +cd "$oracle_dir" +exec cargo run --manifest-path Cargo.toml --release --offline --locked -- "$@" diff --git a/test/conformance/n5/oracles/arrow-rs/scripts/test.sh b/test/conformance/n5/oracles/arrow-rs/scripts/test.sh new file mode 100755 index 0000000..e7bf520 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/scripts/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +export CARGO_NET_OFFLINE=true +cd "$oracle_dir" +exec cargo test --manifest-path Cargo.toml --offline --locked diff --git a/test/conformance/n5/oracles/arrow-rs/src/cases.rs b/test/conformance/n5/oracles/arrow-rs/src/cases.rs new file mode 100644 index 0000000..ef325e8 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/src/cases.rs @@ -0,0 +1,350 @@ +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use parquet::basic::Compression; +use parquet::column::writer::ColumnWriter; +use parquet::data_type::ByteArray; +use parquet::file::properties::{EnabledStatistics, WriterProperties, WriterVersion}; +use parquet::file::writer::SerializedFileWriter; +use parquet::schema::parser::parse_message_type; +use serde_json::{Value, json}; + +use crate::{DynError, Result}; + +pub const DUPLICATE_KEYS: &str = "arrow-rs-duplicate-keys"; +pub const OPTIONAL_KEY_PRESENT: &str = "arrow-rs-optional-key-present"; +pub const LIST_RULE3: &str = "arrow-rs-list-rule3"; +pub const LIST_RULE3_NEAR_NEIGHBOR: &str = "arrow-rs-list-rule3-unannotated-near-neighbor"; + +const DUPLICATE_SCHEMA: &str = r#" +message schema { + OPTIONAL group entries (MAP) { + REPEATED group key_value { + REQUIRED BINARY key (STRING); + OPTIONAL INT32 value; + } + } +} +"#; + +const OPTIONAL_KEY_SCHEMA: &str = r#" +message schema { + OPTIONAL group entries (MAP) { + REPEATED group key_value { + OPTIONAL BINARY key (STRING); + REQUIRED INT32 value; + } + } +} +"#; + +const LIST_RULE3_SCHEMA: &str = r#" +message schema { + OPTIONAL group values (LIST) { + REPEATED group array (LIST) { + REPEATED INT32 array; + } + } +} +"#; + +const LIST_RULE3_NEAR_NEIGHBOR_SCHEMA: &str = r#" +message schema { + OPTIONAL group values (LIST) { + REPEATED group list { + REPEATED INT32 element; + } + } +} +"#; + +const LIST_RULE3_REPETITION: &[i16] = &[0, 0, 0, 0, 2, 1, 1]; +const LIST_RULE3_DEFINITION: &[i16] = &[0, 1, 2, 3, 3, 2, 3]; +const LIST_RULE3_VALUES: &[i32] = &[1, 2, 3]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PageVersion { + V1, + V2, +} + +impl PageVersion { + pub fn label(self) -> &'static str { + match self { + Self::V1 => "v1", + Self::V2 => "v2", + } + } + + fn writer_version(self) -> WriterVersion { + match self { + Self::V1 => WriterVersion::PARQUET_1_0, + Self::V2 => WriterVersion::PARQUET_2_0, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OwnedCase { + DuplicateKeys, + OptionalKeyPresent, + ListRule3, +} + +impl OwnedCase { + pub const ALL: [Self; 3] = [ + Self::DuplicateKeys, + Self::OptionalKeyPresent, + Self::ListRule3, + ]; + + pub fn id(self) -> &'static str { + match self { + Self::DuplicateKeys => DUPLICATE_KEYS, + Self::OptionalKeyPresent => OPTIONAL_KEY_PRESENT, + Self::ListRule3 => LIST_RULE3, + } + } + + fn schema(self) -> &'static str { + match self { + Self::DuplicateKeys => DUPLICATE_SCHEMA, + Self::OptionalKeyPresent => OPTIONAL_KEY_SCHEMA, + Self::ListRule3 => LIST_RULE3_SCHEMA, + } + } + + pub fn file_name(self, version: PageVersion) -> String { + format!("{}_{}.parquet", self.id(), version.label()) + } + + pub fn row_count(self) -> i64 { + match self { + Self::DuplicateKeys => 5, + Self::OptionalKeyPresent => 4, + Self::ListRule3 => 4, + } + } + + pub fn expected_repetitions(self) -> &'static [i16] { + match self { + Self::DuplicateKeys => &[0, 0, 0, 0, 1, 1, 0], + Self::OptionalKeyPresent => &[0, 0, 0, 0, 1], + Self::ListRule3 => LIST_RULE3_REPETITION, + } + } + + pub fn expected_key_definitions(self) -> &'static [i16] { + match self { + Self::DuplicateKeys => &[0, 1, 2, 2, 2, 2, 2], + Self::OptionalKeyPresent => &[0, 1, 3, 3, 3], + Self::ListRule3 => unreachable!("LIST rule 3 has no map key definition stream"), + } + } + + pub fn expected_value_definitions(self) -> &'static [i16] { + match self { + Self::DuplicateKeys => &[0, 1, 2, 3, 3, 3, 3], + Self::OptionalKeyPresent => &[0, 1, 2, 2, 2], + Self::ListRule3 => unreachable!("LIST rule 3 has no map value definition stream"), + } + } + + pub fn expected_key_maximum_definition(self) -> i16 { + match self { + Self::DuplicateKeys => 2, + Self::OptionalKeyPresent => 3, + Self::ListRule3 => unreachable!("LIST rule 3 has no map key definition maximum"), + } + } + + pub fn expected_value_maximum_definition(self) -> i16 { + match self { + Self::DuplicateKeys => 3, + Self::OptionalKeyPresent => 2, + Self::ListRule3 => unreachable!("LIST rule 3 has no map value definition maximum"), + } + } + + pub fn expected_keys(self) -> &'static [&'static str] { + match self { + Self::DuplicateKeys => &["a", "a", "a", "b", "c"], + Self::OptionalKeyPresent => &["a", "b", "c"], + Self::ListRule3 => unreachable!("LIST rule 3 has no map keys"), + } + } + + pub fn expected_values(self) -> &'static [i32] { + match self { + Self::DuplicateKeys => &[1, 2, 3, 4], + Self::OptionalKeyPresent => &[1, 2, 3], + Self::ListRule3 => LIST_RULE3_VALUES, + } + } + + pub fn expected_rows(self) -> Vec { + match self { + Self::DuplicateKeys => vec![ + Value::Null, + json!([]), + json!([{"key": "a", "value": null}]), + json!([ + {"key": "a", "value": 1}, + {"key": "a", "value": 2}, + {"key": "b", "value": 3} + ]), + json!([{"key": "c", "value": 4}]), + ], + Self::OptionalKeyPresent => vec![ + Value::Null, + json!([]), + json!([{"key": "a", "value": 1}]), + json!([ + {"key": "b", "value": 2}, + {"key": "c", "value": 3} + ]), + ], + Self::ListRule3 => vec![ + Value::Null, + json!([]), + json!([[]]), + json!([[1, 2], [], [3]]), + ], + } + } +} + +fn properties(version: PageVersion) -> Arc { + Arc::new( + WriterProperties::builder() + .set_writer_version(version.writer_version()) + .set_created_by("Parquet.jl N5 Arrow Rust oracle 59.2.0".to_owned()) + .set_compression(Compression::UNCOMPRESSED) + .set_dictionary_enabled(false) + .set_statistics_enabled(EnabledStatistics::None) + .set_data_page_size_limit(1024 * 1024) + .set_write_batch_size(1024) + .build(), + ) +} + +fn byte_arrays(values: &[&str]) -> Vec { + values.iter().map(|value| ByteArray::from(*value)).collect() +} + +pub fn write_case(case: OwnedCase, version: PageVersion, path: &Path) -> Result<()> { + if case == OwnedCase::ListRule3 { + return write_list_rule3_schema(LIST_RULE3_SCHEMA, version, path); + } + let schema = Arc::new(parse_message_type(case.schema())?); + let file = File::create(path)?; + let mut writer = SerializedFileWriter::new(file, schema, properties(version))?; + let mut row_group = writer.next_row_group()?; + + let mut key_writer = row_group + .next_column()? + .ok_or_else(|| -> DynError { "schema has no key column".into() })?; + match key_writer.untyped() { + ColumnWriter::ByteArrayColumnWriter(writer) => { + let values = byte_arrays(case.expected_keys()); + let written = writer.write_batch( + &values, + Some(case.expected_key_definitions()), + Some(case.expected_repetitions()), + )?; + if written != values.len() { + return Err( + format!("key writer accepted {written} of {} values", values.len()).into(), + ); + } + } + _ => return Err("key column is not BYTE_ARRAY".into()), + } + key_writer.close()?; + + let mut value_writer = row_group + .next_column()? + .ok_or_else(|| -> DynError { "schema has no value column".into() })?; + match value_writer.untyped() { + ColumnWriter::Int32ColumnWriter(writer) => { + let values = case.expected_values(); + let written = writer.write_batch( + values, + Some(case.expected_value_definitions()), + Some(case.expected_repetitions()), + )?; + if written != values.len() { + return Err( + format!("value writer accepted {written} of {} values", values.len()).into(), + ); + } + } + _ => return Err("value column is not INT32".into()), + } + value_writer.close()?; + if row_group.next_column()?.is_some() { + return Err("schema has more than two physical columns".into()); + } + row_group.close()?; + writer.close()?; + return Ok(()); +} + +pub fn generate_all(directory: &Path) -> Result> { + std::fs::create_dir_all(directory)?; + let mut files = Vec::with_capacity(OwnedCase::ALL.len() * 2); + for case in OwnedCase::ALL { + for version in [PageVersion::V1, PageVersion::V2] { + let path = directory.join(case.file_name(version)); + write_case(case, version, &path)?; + files.push((case, version, path)); + } + } + return Ok(files); +} + +fn write_list_rule3_schema(schema_text: &str, version: PageVersion, path: &Path) -> Result<()> { + let schema = Arc::new(parse_message_type(schema_text)?); + let file = File::create(path)?; + let mut writer = SerializedFileWriter::new(file, schema, properties(version))?; + let mut row_group = writer.next_row_group()?; + let mut column = row_group + .next_column()? + .ok_or_else(|| -> DynError { "rule-3 schema has no physical column".into() })?; + match column.untyped() { + ColumnWriter::Int32ColumnWriter(writer) => { + let written = writer.write_batch( + LIST_RULE3_VALUES, + Some(LIST_RULE3_DEFINITION), + Some(LIST_RULE3_REPETITION), + )?; + if written != LIST_RULE3_VALUES.len() { + return Err("rule-3 writer did not consume every dense value".into()); + } + } + _ => return Err("rule-3 leaf is not INT32".into()), + } + column.close()?; + if row_group.next_column()?.is_some() { + return Err("rule-3 schema has more than one physical column".into()); + } + row_group.close()?; + writer.close()?; + return Ok(()); +} + +pub fn generate_list_rule3_near_neighbor(directory: &Path) -> Result> { + std::fs::create_dir_all(directory)?; + let mut files = Vec::with_capacity(2); + for version in [PageVersion::V1, PageVersion::V2] { + let path = directory.join(format!( + "{}_{}.parquet", + LIST_RULE3_NEAR_NEIGHBOR, + version.label() + )); + write_list_rule3_schema(LIST_RULE3_NEAR_NEIGHBOR_SCHEMA, version, &path)?; + files.push((version, path)); + } + return Ok(files); +} diff --git a/test/conformance/n5/oracles/arrow-rs/src/evidence.rs b/test/conformance/n5/oracles/arrow-rs/src/evidence.rs new file mode 100644 index 0000000..89f718c --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/src/evidence.rs @@ -0,0 +1,1129 @@ +use std::collections::BTreeMap; +use std::fmt::Write as FmtWrite; +use std::fs::File; +use std::io::Read; +use std::path::Path; +use std::process::Command; + +use arrow_array::{ + Array, BinaryArray, BooleanArray, FixedSizeListArray, Float32Array, Float64Array, Int8Array, + Int16Array, Int32Array, Int64Array, LargeBinaryArray, LargeListArray, LargeStringArray, + ListArray, MapArray, StringArray, StructArray, UInt8Array, UInt16Array, UInt32Array, + UInt64Array, +}; +use arrow_cast::display::array_value_to_string; +use arrow_json::WriterBuilder; +use arrow_json::writer::LineDelimited; +use arrow_schema::{DataType, Field}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::column::page::{Page, PageReader}; +use parquet::column::reader::ColumnReader; +use parquet::data_type::{ByteArray, FixedLenByteArray, Int96}; +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::schema::printer::print_schema; +use parquet::schema::types::ColumnDescriptor; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use crate::cases::{LIST_RULE3, LIST_RULE3_NEAR_NEIGHBOR, OwnedCase, PageVersion}; +use crate::{DynError, Result}; + +pub const ARROW_RS_VERSION: &str = "59.2.0"; +pub const ARROW_RS_COMMIT: &str = "782e5a685501a9db6cc8e9a3b7cbff894940c47a"; +pub const RUST_TOOLCHAIN: &str = "1.96.1"; +pub const RUSTC_VERSION_OUTPUT: &str = "rustc 1.96.1 (31fca3adb 2026-06-26)"; +pub const CARGO_VERSION_OUTPUT: &str = "cargo 1.96.1 (356927216 2026-06-26)"; + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct OracleEvidence { + pub evidence_version: u32, + pub oracle: ToolEvidence, + pub action: String, + pub files: Vec, +} + +impl OracleEvidence { + pub fn new(action: &str, files: Vec) -> Result { + return Ok(Self { + evidence_version: 1, + oracle: ToolEvidence::current()?, + action: action.to_owned(), + files, + }); + } +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ToolEvidence { + pub name: String, + pub version: String, + pub commit: String, + pub rust_toolchain: String, + pub rustc: String, + pub cargo: String, +} + +impl ToolEvidence { + pub(crate) fn current() -> Result { + return Self::current_with(command_version); + } + + fn current_with(mut version: F) -> Result + where + F: FnMut(&str) -> Result, + { + let rustc = version("rustc") + .map_err(|error| format!("cannot obtain required rustc version: {error}"))?; + if rustc != RUSTC_VERSION_OUTPUT { + return Err(format!( + "rustc version differs from pin: expected {RUSTC_VERSION_OUTPUT:?}, got {rustc:?}" + ) + .into()); + } + let cargo = version("cargo") + .map_err(|error| format!("cannot obtain required cargo version: {error}"))?; + if cargo != CARGO_VERSION_OUTPUT { + return Err(format!( + "cargo version differs from pin: expected {CARGO_VERSION_OUTPUT:?}, got {cargo:?}" + ) + .into()); + } + return Ok(Self { + name: "arrow-rs".to_owned(), + version: ARROW_RS_VERSION.to_owned(), + commit: ARROW_RS_COMMIT.to_owned(), + rust_toolchain: RUST_TOOLCHAIN.to_owned(), + rustc, + cargo, + }); + } +} + +fn command_version(program: &str) -> Result { + let output = Command::new(program) + .arg("--version") + .output() + .map_err(|error| format!("cannot execute {program} --version: {error}"))?; + if !output.status.success() { + return Err(format!("{program} --version failed with {}", output.status).into()); + } + let stdout = String::from_utf8(output.stdout) + .map_err(|error| format!("{program} --version returned non-UTF-8 output: {error}"))?; + let value = stdout.trim(); + if value.is_empty() { + return Err(format!("{program} --version returned empty output").into()); + } + return Ok(value.to_owned()); +} + +#[cfg(test)] +mod tool_evidence_tests { + use std::io::{Error, ErrorKind}; + + use super::*; + + #[test] + fn tool_evidence_rejects_missing_or_different_versions() -> Result<()> { + let correct = ToolEvidence::current_with(|program| { + return Ok(match program { + "rustc" => RUSTC_VERSION_OUTPUT, + "cargo" => CARGO_VERSION_OUTPUT, + _ => return Err(format!("unexpected program {program}").into()), + } + .to_owned()); + })?; + assert_eq!(correct.rustc, RUSTC_VERSION_OUTPUT); + assert_eq!(correct.cargo, CARGO_VERSION_OUTPUT); + + let missing_rustc = ToolEvidence::current_with(|_| { + return Err(Error::new(ErrorKind::NotFound, "injected missing tool").into()); + }) + .unwrap_err(); + assert!( + missing_rustc + .to_string() + .contains("cannot obtain required rustc version") + ); + + let wrong_rustc = ToolEvidence::current_with(|program| { + return Ok(match program { + "rustc" => "rustc 0.0.0 (wrong 1970-01-01)", + "cargo" => CARGO_VERSION_OUTPUT, + _ => return Err(format!("unexpected program {program}").into()), + } + .to_owned()); + }) + .unwrap_err(); + assert!( + wrong_rustc + .to_string() + .contains("rustc version differs from pin") + ); + + let missing_cargo = ToolEvidence::current_with(|program| { + if program == "rustc" { + return Ok(RUSTC_VERSION_OUTPUT.to_owned()); + } + return Err(Error::new(ErrorKind::NotFound, "injected missing tool").into()); + }) + .unwrap_err(); + assert!( + missing_cargo + .to_string() + .contains("cannot obtain required cargo version") + ); + + let wrong_cargo = ToolEvidence::current_with(|program| { + return Ok(match program { + "rustc" => RUSTC_VERSION_OUTPUT, + "cargo" => "cargo 0.0.0 (wrong 1970-01-01)", + _ => return Err(format!("unexpected program {program}").into()), + } + .to_owned()); + }) + .unwrap_err(); + assert!( + wrong_cargo + .to_string() + .contains("cargo version differs from pin") + ); + return Ok(()); + } +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct FileEvidence { + pub case_id: String, + pub file_name: String, + pub sha256: String, + pub file_bytes: u64, + pub rows: i64, + pub row_groups: usize, + pub physical_schema: String, + pub columns: Vec, + pub arrow: ArrowEvidence, +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ColumnEvidence { + pub path: Vec, + pub physical_type: String, + pub maximum_definition_level: i16, + pub maximum_repetition_level: i16, + pub row_groups: Vec, +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ColumnRowGroupEvidence { + pub row_group: usize, + pub rows: usize, + pub compression: String, + pub repetition: Vec, + pub definition: Vec, + pub dense_values: Vec, + pub pages: Vec, +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct PageEvidence { + pub kind: String, + pub version: Option, + pub encoding: String, + pub values: u32, + pub rows: Option, +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ArrowEvidence { + pub status: String, + pub schema: Option>, + pub canonical_rows: Vec, + pub json_rows: Vec, + pub ordered_map_rows: Option>, + pub diagnostic: Option, +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ArrowFieldEvidence { + pub name: String, + pub nullable: bool, + pub data_type: Value, + pub metadata: BTreeMap, +} + +#[derive(Debug)] +struct RawPage { + evidence: PageEvidence, + declared_rows: Option, +} + +#[derive(Debug)] +struct ColumnRead { + records: usize, + levels: usize, + repetition: Vec, + definition: Vec, + values: Vec, +} + +fn sha256(path: &Path) -> Result { + let mut file = File::open(path)?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + return Ok(format!("{:x}", digest.finalize())); +} + +fn byte_value(bytes: &[u8]) -> Value { + match std::str::from_utf8(bytes) { + Ok(value) => Value::String(value.to_owned()), + Err(_) => { + let mut hex = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(&mut hex, "{byte:02x}").expect("writing to String cannot fail"); + } + json!({"hex": hex}) + } + } +} + +fn int96_value(value: &Int96) -> Value { + return Value::Array(value.data().iter().copied().map(Value::from).collect()); +} + +fn fixed_value(value: &FixedLenByteArray) -> Value { + return byte_value(value.data()); +} + +fn read_column( + reader: ColumnReader, + rows: usize, + descriptor: &ColumnDescriptor, +) -> Result { + let maximum_definition = descriptor.max_def_level(); + let maximum_repetition = descriptor.max_rep_level(); + let mut definition = Vec::new(); + let mut repetition = Vec::new(); + + macro_rules! read_typed { + ($reader:expr, $value_type:ty, $convert:expr) => {{ + let mut reader = $reader; + let mut values: Vec<$value_type> = Vec::new(); + let (records, values_read, levels) = if rows == 0 { + (0, 0, 0) + } else { + reader.read_records( + rows, + (maximum_definition > 0).then_some(&mut definition), + (maximum_repetition > 0).then_some(&mut repetition), + &mut values, + )? + }; + if values_read != values.len() { + return Err("column reader value count differs from its output".into()); + } + let converted = values.iter().map($convert).collect(); + (records, levels, converted) + }}; + } + + let (records, levels, values) = match reader { + ColumnReader::BoolColumnReader(reader) => { + read_typed!(reader, bool, |value: &bool| Value::Bool(*value)) + } + ColumnReader::Int32ColumnReader(reader) => { + read_typed!(reader, i32, |value: &i32| Value::from(*value)) + } + ColumnReader::Int64ColumnReader(reader) => { + read_typed!(reader, i64, |value: &i64| Value::from(*value)) + } + ColumnReader::Int96ColumnReader(reader) => { + read_typed!(reader, Int96, |value: &Int96| int96_value(value)) + } + ColumnReader::FloatColumnReader(reader) => read_typed!(reader, f32, |value: &f32| { + Value::String(format!("0x{:08x}", value.to_bits())) + }), + ColumnReader::DoubleColumnReader(reader) => read_typed!(reader, f64, |value: &f64| { + Value::String(format!("0x{:016x}", value.to_bits())) + }), + ColumnReader::ByteArrayColumnReader(reader) => { + read_typed!(reader, ByteArray, |value: &ByteArray| byte_value( + value.data() + )) + } + ColumnReader::FixedLenByteArrayColumnReader(reader) => { + read_typed!(reader, FixedLenByteArray, |value: &FixedLenByteArray| { + fixed_value(value) + }) + } + }; + if maximum_definition == 0 { + definition.resize(levels, 0); + } + if maximum_repetition == 0 { + repetition.resize(levels, 0); + } + if definition.len() != levels || repetition.len() != levels { + return Err("column reader level output has an inconsistent length".into()); + } + return Ok(ColumnRead { + records, + levels, + repetition, + definition, + values, + }); +} + +fn read_pages(mut reader: Box) -> Result> { + let mut pages = Vec::new(); + while let Some(page) = reader.get_next_page()? { + let raw = match page { + Page::DictionaryPage { + num_values, + encoding, + .. + } => RawPage { + evidence: PageEvidence { + kind: "dictionary".to_owned(), + version: None, + encoding: format!("{encoding:?}"), + values: num_values, + rows: None, + }, + declared_rows: None, + }, + Page::DataPage { + num_values, + encoding, + .. + } => RawPage { + evidence: PageEvidence { + kind: "data".to_owned(), + version: Some("v1".to_owned()), + encoding: format!("{encoding:?}"), + values: num_values, + rows: None, + }, + declared_rows: None, + }, + Page::DataPageV2 { + num_values, + num_rows, + encoding, + .. + } => RawPage { + evidence: PageEvidence { + kind: "data".to_owned(), + version: Some("v2".to_owned()), + encoding: format!("{encoding:?}"), + values: num_values, + rows: None, + }, + declared_rows: Some(num_rows as usize), + }, + }; + pages.push(raw); + } + return Ok(pages); +} + +fn finish_pages(mut pages: Vec, repetition: &[i16]) -> Result> { + let mut offset = 0_usize; + for page in &mut pages { + if page.evidence.kind != "data" { + continue; + } + let end = offset + .checked_add(page.evidence.values as usize) + .ok_or_else(|| -> DynError { "page level count overflow".into() })?; + let levels = repetition + .get(offset..end) + .ok_or_else(|| -> DynError { "page levels exceed column output".into() })?; + let rows = levels.iter().filter(|level| **level == 0).count(); + if let Some(declared) = page.declared_rows { + if declared != rows { + return Err(format!( + "V2 page declares {declared} rows but repetition levels contain {rows}" + ) + .into()); + } + } + page.evidence.rows = Some(rows); + offset = end; + } + if offset != repetition.len() { + return Err("data-page value counts do not span the column levels".into()); + } + return Ok(pages.into_iter().map(|page| page.evidence).collect()); +} + +fn schema_text(reader: &SerializedFileReader) -> Result { + let mut bytes = Vec::new(); + print_schema( + &mut bytes, + reader + .metadata() + .file_metadata() + .schema_descr() + .root_schema(), + ); + return Ok(String::from_utf8(bytes)?); +} + +fn field_evidence(field: &Field) -> ArrowFieldEvidence { + let metadata = field + .metadata() + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + ArrowFieldEvidence { + name: field.name().to_owned(), + nullable: field.is_nullable(), + data_type: data_type_evidence(field.data_type()), + metadata, + } +} + +fn data_type_evidence(data_type: &DataType) -> Value { + match data_type { + DataType::List(field) => json!({"list": field_evidence(field)}), + DataType::LargeList(field) => json!({"large_list": field_evidence(field)}), + DataType::FixedSizeList(field, size) => { + json!({"fixed_size_list": {"size": size, "field": field_evidence(field)}}) + } + DataType::Struct(fields) => json!({ + "struct": fields.iter().map(|field| field_evidence(field)).collect::>() + }), + DataType::Map(field, sorted) => { + json!({"map": {"sorted": sorted, "entries": field_evidence(field)}}) + } + DataType::Dictionary(key, value) => json!({ + "dictionary": { + "key": data_type_evidence(key), + "value": data_type_evidence(value) + } + }), + other => Value::String(format!("{other:?}")), + } +} + +fn canonical_value(array: &dyn Array, index: usize) -> Result { + if array.is_null(index) { + return Ok(Value::Null); + } + let value = match array.data_type() { + DataType::Boolean => Value::Bool( + array + .as_any() + .downcast_ref::() + .ok_or("Boolean array downcast failed")? + .value(index), + ), + DataType::Int8 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("Int8 array downcast failed")? + .value(index), + ), + DataType::Int16 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("Int16 array downcast failed")? + .value(index), + ), + DataType::Int32 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("Int32 array downcast failed")? + .value(index), + ), + DataType::Int64 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("Int64 array downcast failed")? + .value(index), + ), + DataType::UInt8 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("UInt8 array downcast failed")? + .value(index), + ), + DataType::UInt16 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("UInt16 array downcast failed")? + .value(index), + ), + DataType::UInt32 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("UInt32 array downcast failed")? + .value(index), + ), + DataType::UInt64 => Value::from( + array + .as_any() + .downcast_ref::() + .ok_or("UInt64 array downcast failed")? + .value(index), + ), + DataType::Float32 => Value::String(format!( + "0x{:08x}", + array + .as_any() + .downcast_ref::() + .ok_or("Float32 array downcast failed")? + .value(index) + .to_bits() + )), + DataType::Float64 => Value::String(format!( + "0x{:016x}", + array + .as_any() + .downcast_ref::() + .ok_or("Float64 array downcast failed")? + .value(index) + .to_bits() + )), + DataType::Utf8 => Value::String( + array + .as_any() + .downcast_ref::() + .ok_or("Utf8 array downcast failed")? + .value(index) + .to_owned(), + ), + DataType::LargeUtf8 => Value::String( + array + .as_any() + .downcast_ref::() + .ok_or("LargeUtf8 array downcast failed")? + .value(index) + .to_owned(), + ), + DataType::Binary => byte_value( + array + .as_any() + .downcast_ref::() + .ok_or("Binary array downcast failed")? + .value(index), + ), + DataType::LargeBinary => byte_value( + array + .as_any() + .downcast_ref::() + .ok_or("LargeBinary array downcast failed")? + .value(index), + ), + DataType::List(_) => { + let list = array + .as_any() + .downcast_ref::() + .ok_or("List array downcast failed")?; + let child = list.value(index); + Value::Array( + (0..child.len()) + .map(|child_index| canonical_value(child.as_ref(), child_index)) + .collect::>>()?, + ) + } + DataType::LargeList(_) => { + let list = array + .as_any() + .downcast_ref::() + .ok_or("LargeList array downcast failed")?; + let child = list.value(index); + Value::Array( + (0..child.len()) + .map(|child_index| canonical_value(child.as_ref(), child_index)) + .collect::>>()?, + ) + } + DataType::FixedSizeList(_, _) => { + let list = array + .as_any() + .downcast_ref::() + .ok_or("FixedSizeList array downcast failed")?; + let child = list.value(index); + Value::Array( + (0..child.len()) + .map(|child_index| canonical_value(child.as_ref(), child_index)) + .collect::>>()?, + ) + } + DataType::Struct(fields) => { + let structure = array + .as_any() + .downcast_ref::() + .ok_or("Struct array downcast failed")?; + Value::Array( + fields + .iter() + .zip(structure.columns()) + .map(|(field, child)| { + Ok(json!({ + "field": field.name(), + "value": canonical_value(child.as_ref(), index)? + })) + }) + .collect::>>()?, + ) + } + DataType::Map(_, _) => { + let map = array + .as_any() + .downcast_ref::() + .ok_or("Map array downcast failed")?; + let offsets = map.value_offsets(); + let start = offsets[index] as usize; + let stop = offsets[index + 1] as usize; + let entries = map.entries(); + let mut pairs = Vec::with_capacity(stop - start); + for entry in start..stop { + pairs.push(json!({ + "key": canonical_value(entries.column(0).as_ref(), entry)?, + "value": canonical_value(entries.column(1).as_ref(), entry)? + })); + } + Value::Array(pairs) + } + _ => json!({ + "data_type": format!("{:?}", array.data_type()), + "display": array_value_to_string(array, index)? + }), + }; + return Ok(value); +} + +fn canonical_rows(batches: &[arrow_array::RecordBatch]) -> Result> { + let mut rows = Vec::new(); + for batch in batches { + for row in 0..batch.num_rows() { + let fields = batch + .schema() + .fields() + .iter() + .zip(batch.columns()) + .map(|(field, column)| { + Ok(json!({ + "field": field.name(), + "value": canonical_value(column.as_ref(), row)? + })) + }) + .collect::>>()?; + rows.push(Value::Array(fields)); + } + } + return Ok(rows); +} + +fn ordered_map_rows(batches: &[arrow_array::RecordBatch]) -> Result>> { + if batches.first().is_none_or(|batch| batch.num_columns() != 1) { + return Ok(None); + } + let mut rows = Vec::new(); + for batch in batches { + if !matches!(batch.column(0).data_type(), DataType::Map(_, _)) { + return Ok(None); + } + for row in 0..batch.num_rows() { + rows.push(canonical_value(batch.column(0).as_ref(), row)?); + } + } + return Ok(Some(rows)); +} + +fn read_arrow(path: &Path) -> Result { + let builder = ParquetRecordBatchReaderBuilder::try_new(File::open(path)?)?; + let schema = builder.schema().as_ref().clone(); + let mut reader = builder.with_batch_size(1024).build()?; + let mut batches = Vec::new(); + for batch in &mut reader { + batches.push(batch?); + } + let canonical_rows = canonical_rows(&batches)?; + let ordered_map_rows = ordered_map_rows(&batches)?; + let mut json_bytes = Vec::new(); + let json_result = { + let mut writer = WriterBuilder::new() + .with_explicit_nulls(true) + .build::<_, LineDelimited>(&mut json_bytes); + let mut result = Ok(()); + for batch in &batches { + if let Err(error) = writer.write(batch) { + result = Err(error); + break; + } + } + if result.is_ok() { + result = writer.finish(); + } + result + }; + let (json_rows, diagnostic) = match json_result { + Ok(()) => { + let text = String::from_utf8(json_bytes)?; + (text.lines().map(str::to_owned).collect(), None) + } + Err(error) => ( + Vec::new(), + Some(format!("Arrow JSON diagnostic unavailable: {error}")), + ), + }; + let fields = schema + .fields() + .iter() + .map(|field| field_evidence(field)) + .collect(); + return Ok(ArrowEvidence { + status: "ok".to_owned(), + schema: Some(fields), + canonical_rows, + json_rows, + ordered_map_rows, + diagnostic, + }); +} + +fn arrow_evidence(path: &Path) -> ArrowEvidence { + match read_arrow(path) { + Ok(evidence) => evidence, + Err(error) => ArrowEvidence { + status: "unsupported".to_owned(), + schema: None, + canonical_rows: Vec::new(), + json_rows: Vec::new(), + ordered_map_rows: None, + diagnostic: Some(error.to_string()), + }, + } +} + +pub fn inspect_file(path: &Path, case_id: &str) -> Result { + let file = File::open(path)?; + let reader = SerializedFileReader::new(file)?; + let metadata = reader.metadata(); + let file_metadata = metadata.file_metadata(); + let schema_descriptor = file_metadata.schema_descr(); + let mut columns = Vec::with_capacity(schema_descriptor.num_columns()); + for column_index in 0..schema_descriptor.num_columns() { + let descriptor = schema_descriptor.column(column_index); + let mut groups = Vec::with_capacity(metadata.num_row_groups()); + for row_group_index in 0..metadata.num_row_groups() { + let group = reader.get_row_group(row_group_index)?; + let rows = usize::try_from(group.metadata().num_rows())?; + let compression = format!( + "{:?}", + group.metadata().column(column_index).compression_codec() + ); + let raw_pages = read_pages(group.get_column_page_reader(column_index)?)?; + let output = read_column(group.get_column_reader(column_index)?, rows, &descriptor)?; + if output.records != rows { + return Err(format!( + "column {} row group {row_group_index} read {} of {rows} rows", + descriptor.path(), + output.records + ) + .into()); + } + if output.levels != output.repetition.len() { + return Err("column level count differs from repetitions".into()); + } + let pages = finish_pages(raw_pages, &output.repetition)?; + groups.push(ColumnRowGroupEvidence { + row_group: row_group_index, + rows, + compression, + repetition: output.repetition, + definition: output.definition, + dense_values: output.values, + pages, + }); + } + columns.push(ColumnEvidence { + path: descriptor.path().parts().to_vec(), + physical_type: format!("{:?}", descriptor.physical_type()), + maximum_definition_level: descriptor.max_def_level(), + maximum_repetition_level: descriptor.max_rep_level(), + row_groups: groups, + }); + } + let file_name = path + .file_name() + .ok_or_else(|| -> DynError { "input path has no file name".into() })? + .to_string_lossy() + .into_owned(); + return Ok(FileEvidence { + case_id: case_id.to_owned(), + file_name, + sha256: sha256(path)?, + file_bytes: std::fs::metadata(path)?.len(), + rows: file_metadata.num_rows(), + row_groups: metadata.num_row_groups(), + physical_schema: schema_text(&reader)?, + columns, + arrow: arrow_evidence(path), + }); +} + +fn expected_dense_strings(values: &[&str]) -> Vec { + return values + .iter() + .map(|value| Value::String((*value).to_owned())) + .collect(); +} + +fn expected_dense_i32(values: &[i32]) -> Vec { + return values.iter().copied().map(Value::from).collect(); +} + +fn expected_arrow_type(case: OwnedCase) -> Value { + let value_nullable = case == OwnedCase::DuplicateKeys; + return json!({ + "map": { + "sorted": false, + "entries": { + "name": "key_value", + "nullable": false, + "metadata": {}, + "data_type": { + "struct": [ + { + "name": "key", + "nullable": false, + "metadata": {}, + "data_type": "Utf8" + }, + { + "name": "value", + "nullable": value_nullable, + "metadata": {}, + "data_type": "Int32" + } + ] + } + } + } + }); +} + +fn expected_canonical_rows(field: &str, values: Vec) -> Vec { + return values + .into_iter() + .map(|value| json!([{"field": field, "value": value}])) + .collect(); +} + +pub fn verify_owned(case: OwnedCase, version: PageVersion, file: &FileEvidence) -> Result<()> { + if case == OwnedCase::ListRule3 { + return verify_list_rule3(version, file); + } + if file.case_id != case.id() || file.file_name != case.file_name(version) { + return Err("owned fixture identity differs from its case".into()); + } + if file.rows != case.row_count() || file.row_groups != 1 || file.columns.len() != 2 { + return Err("owned fixture row, row-group, or column count differs".into()); + } + let expected_paths = [ + ["entries", "key_value", "key"], + ["entries", "key_value", "value"], + ]; + for (column, expected_path) in file.columns.iter().zip(expected_paths) { + let expected: Vec = expected_path + .iter() + .map(|value| (*value).to_owned()) + .collect(); + if column.path != expected || column.row_groups.len() != 1 { + return Err("owned fixture leaf path or row-group evidence differs".into()); + } + let group = &column.row_groups[0]; + if group.rows != case.row_count() as usize + || group.compression != "UNCOMPRESSED" + || group.repetition != case.expected_repetitions() + { + return Err("owned fixture repetition stream differs".into()); + } + let page_versions: Vec<&str> = group + .pages + .iter() + .filter_map(|page| page.version.as_deref()) + .collect(); + if page_versions.is_empty() + || page_versions + .iter() + .any(|actual| *actual != version.label()) + { + return Err("owned fixture data-page version differs".into()); + } + } + let keys = &file.columns[0].row_groups[0]; + let values = &file.columns[1].row_groups[0]; + if file.columns[0].physical_type != "BYTE_ARRAY" + || file.columns[0].maximum_definition_level != case.expected_key_maximum_definition() + || file.columns[0].maximum_repetition_level != 1 + || file.columns[1].physical_type != "INT32" + || file.columns[1].maximum_definition_level != case.expected_value_maximum_definition() + || file.columns[1].maximum_repetition_level != 1 + || keys.definition != case.expected_key_definitions() + || keys.dense_values != expected_dense_strings(case.expected_keys()) + || values.definition != case.expected_value_definitions() + || values.dense_values != expected_dense_i32(case.expected_values()) + { + return Err("owned fixture definitions or dense values differ".into()); + } + let schema = file + .arrow + .schema + .as_ref() + .filter(|schema| schema.len() == 1) + .ok_or_else(|| -> DynError { "owned fixture has no exact Arrow schema".into() })?; + if file.arrow.status != "ok" + || schema[0].name != "entries" + || !schema[0].nullable + || !schema[0].metadata.is_empty() + || schema[0].data_type != expected_arrow_type(case) + || file.arrow.canonical_rows != expected_canonical_rows("entries", case.expected_rows()) + || file.arrow.ordered_map_rows.as_ref() != Some(&case.expected_rows()) + { + return Err("owned fixture Arrow schema or ordered rows differ".into()); + } + return Ok(()); +} + +fn expected_list_rule3_arrow_type(element_name: &str) -> Value { + return json!({ + "list": { + "name": element_name, + "nullable": false, + "metadata": {}, + "data_type": { + "list": { + "name": element_name, + "nullable": false, + "metadata": {}, + "data_type": "Int32" + } + } + } + }); +} + +fn verify_list_rule3_case( + case_id: &str, + path: [&str; 3], + arrow_element_name: &str, + physical_schema: &str, + version: PageVersion, + file: &FileEvidence, +) -> Result<()> { + let expected_name = format!("{}_{}.parquet", case_id, version.label()); + if file.case_id != case_id + || file.file_name != expected_name + || file.rows != 4 + || file.row_groups != 1 + || file.columns.len() != 1 + || file.physical_schema != physical_schema + { + return Err("rule-3 fixture identity or shape differs".into()); + } + let column = &file.columns[0]; + let group = column + .row_groups + .first() + .ok_or_else(|| -> DynError { "rule-3 fixture has no row-group evidence".into() })?; + if column.path != path + || column.physical_type != "INT32" + || column.maximum_definition_level != 3 + || column.maximum_repetition_level != 2 + || group.rows != 4 + || group.compression != "UNCOMPRESSED" + || group.repetition != [0, 0, 0, 0, 2, 1, 1] + || group.definition != [0, 1, 2, 3, 3, 2, 3] + || group.dense_values != [Value::from(1), Value::from(2), Value::from(3)] + { + return Err("rule-3 physical evidence differs".into()); + } + let page_versions: Vec<&str> = group + .pages + .iter() + .filter_map(|page| page.version.as_deref()) + .collect(); + if page_versions.is_empty() + || page_versions + .iter() + .any(|actual| *actual != version.label()) + { + return Err("rule-3 page version differs".into()); + } + let expected_rows = [ + "{\"values\":null}", + "{\"values\":[]}", + "{\"values\":[[]]}", + "{\"values\":[[1,2],[],[3]]}", + ]; + let expected_values = vec![ + Value::Null, + json!([]), + json!([[]]), + json!([[1, 2], [], [3]]), + ]; + let schema = file + .arrow + .schema + .as_ref() + .filter(|schema| schema.len() == 1) + .ok_or_else(|| -> DynError { "rule-3 fixture has no exact Arrow schema".into() })?; + if file.arrow.status != "ok" + || schema[0].name != "values" + || !schema[0].nullable + || !schema[0].metadata.is_empty() + || schema[0].data_type != expected_list_rule3_arrow_type(arrow_element_name) + || file.arrow.canonical_rows != expected_canonical_rows("values", expected_values) + || file.arrow.json_rows != expected_rows + || file.arrow.ordered_map_rows.is_some() + || file.arrow.diagnostic.is_some() + { + return Err("Arrow RecordBatch did not report the expected rule-3 outcome".into()); + } + return Ok(()); +} + +pub fn verify_list_rule3(version: PageVersion, file: &FileEvidence) -> Result<()> { + const PHYSICAL_SCHEMA: &str = "message schema {\n OPTIONAL group values (LIST) {\n REPEATED group array (LIST) {\n REPEATED INT32 array;\n }\n }\n}\n"; + return verify_list_rule3_case( + LIST_RULE3, + ["values", "array", "array"], + "array", + PHYSICAL_SCHEMA, + version, + file, + ); +} + +pub fn verify_list_rule3_near_neighbor(version: PageVersion, file: &FileEvidence) -> Result<()> { + const PHYSICAL_SCHEMA: &str = "message schema {\n OPTIONAL group values (LIST) {\n REPEATED group list {\n REPEATED INT32 element;\n }\n }\n}\n"; + return verify_list_rule3_case( + LIST_RULE3_NEAR_NEIGHBOR, + ["values", "list", "element"], + "element", + PHYSICAL_SCHEMA, + version, + file, + ); +} diff --git a/test/conformance/n5/oracles/arrow-rs/src/main.rs b/test/conformance/n5/oracles/arrow-rs/src/main.rs new file mode 100644 index 0000000..0e0bfd5 --- /dev/null +++ b/test/conformance/n5/oracles/arrow-rs/src/main.rs @@ -0,0 +1,454 @@ +mod cases; +mod evidence; + +use std::env; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::panic::{self, AssertUnwindSafe}; +use std::path::{Path, PathBuf}; + +use cases::{LIST_RULE3_NEAR_NEIGHBOR, generate_all, generate_list_rule3_near_neighbor}; +use evidence::{ + FileEvidence, OracleEvidence, ToolEvidence, inspect_file, verify_list_rule3_near_neighbor, + verify_owned, +}; + +pub type DynError = Box; +pub type Result = std::result::Result; + +fn usage() -> &'static str { + "usage:\n arrow-rs-oracle generate --output DIR [--evidence FILE]\n arrow-rs-oracle diagnose-rule3-near-neighbor --output DIR [--evidence FILE]\n arrow-rs-oracle inspect --input FILE --case-id ID [--evidence FILE]\n arrow-rs-oracle audit --input FILE_OR_DIR [--evidence FILE]\n arrow-rs-oracle verify --input FILE --expected FILE --case-id ID [--evidence FILE]\n arrow-rs-oracle versions [--evidence FILE]" +} + +fn option(args: &[String], name: &str) -> Result> { + let mut value = None; + let mut index = 0; + while index < args.len() { + if args[index] == name { + if value.is_some() { + return Err(format!("duplicate option {name}").into()); + } + let next = args + .get(index + 1) + .ok_or_else(|| format!("missing value for {name}"))?; + value = Some(next.clone()); + index += 2; + } else { + index += 1; + } + } + return Ok(value); +} + +fn required(args: &[String], name: &str) -> Result { + return option(args, name)?.ok_or_else(|| format!("required option {name} is absent").into()); +} + +fn validate_options(args: &[String], allowed: &[&str]) -> Result<()> { + let mut index = 0; + while index < args.len() { + let current = &args[index]; + if !allowed.contains(¤t.as_str()) { + return Err(format!("unknown option {current}").into()); + } + if index + 1 >= args.len() { + return Err(format!("missing value for {current}").into()); + } + index += 2; + } + return Ok(()); +} + +fn output_evidence(evidence: &OracleEvidence, path: Option) -> Result<()> { + match path { + Some(path) => { + let mut file = File::create(path)?; + serde_json::to_writer_pretty(&mut file, evidence)?; + file.write_all(b"\n")?; + } + None => { + let stdout = io::stdout(); + let mut output = stdout.lock(); + serde_json::to_writer_pretty(&mut output, evidence)?; + output.write_all(b"\n")?; + } + } + return Ok(()); +} + +#[derive(serde::Serialize)] +struct AuditEvidence { + evidence_version: u32, + oracle: ToolEvidence, + action: String, + file_count: usize, + supported_count: usize, + unsupported_count: usize, + files: Vec, +} + +#[derive(serde::Serialize)] +struct AuditFileEvidence { + status: String, + file: String, + evidence: Option, + error: Option, +} + +fn output_audit(evidence: &AuditEvidence, path: Option) -> Result<()> { + match path { + Some(path) => { + let mut file = File::create(path)?; + serde_json::to_writer_pretty(&mut file, evidence)?; + file.write_all(b"\n")?; + } + None => { + let stdout = io::stdout(); + let mut output = stdout.lock(); + serde_json::to_writer_pretty(&mut output, evidence)?; + output.write_all(b"\n")?; + } + } + return Ok(()); +} + +fn parquet_files(root: &Path) -> Result> { + let metadata = fs::symlink_metadata(root)?; + if metadata.file_type().is_symlink() { + return Err("audit input must not be a symbolic link".into()); + } + if metadata.is_file() { + if root.extension().and_then(|value| value.to_str()) != Some("parquet") { + return Err("audit input file must have a .parquet suffix".into()); + } + return Ok(vec![root.to_path_buf()]); + } + if !metadata.is_dir() { + return Err("audit input is not a file or directory".into()); + } + let mut pending = vec![root.to_path_buf()]; + let mut files = Vec::new(); + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_symlink() { + return Err( + format!("audit input contains symbolic link {}", path.display()).into(), + ); + } + if metadata.is_dir() { + pending.push(path); + } else if metadata.is_file() + && path.extension().and_then(|value| value.to_str()) == Some("parquet") + { + files.push(path); + } + } + } + files.sort_by(|left, right| audit_name(root, left).cmp(&audit_name(root, right))); + if files.is_empty() { + return Err("audit input contains no .parquet files".into()); + } + return Ok(files); +} + +fn audit_name(root: &Path, file: &Path) -> String { + if root.is_file() { + return file + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_owned(); + } + return file + .strip_prefix(root) + .unwrap_or(file) + .components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/"); +} + +fn audit(args: &[String]) -> Result { + validate_options(args, &["--input", "--evidence"])?; + let root = PathBuf::from(required(args, "--input")?); + let files = parquet_files(&root)?; + let mut results = Vec::with_capacity(files.len()); + let mut supported = 0_usize; + let mut unsupported = 0_usize; + let panic_hook = panic::take_hook(); + panic::set_hook(Box::new(|_| {})); + for file in files { + let name = audit_name(&root, &file); + let inspected = panic::catch_unwind(AssertUnwindSafe(|| inspect_file(&file, &name))); + match inspected { + Ok(Ok(evidence)) => { + results.push(AuditFileEvidence { + status: "supported".to_owned(), + file: name, + evidence: Some(evidence), + error: None, + }); + supported += 1; + } + Ok(Err(error)) => { + let path = file.to_string_lossy(); + let message = error.to_string().replace(path.as_ref(), ""); + results.push(AuditFileEvidence { + status: "unsupported".to_owned(), + file: name, + evidence: None, + error: Some(message), + }); + unsupported += 1; + } + Err(payload) => { + let message = if let Some(value) = payload.downcast_ref::() { + value.clone() + } else if let Some(value) = payload.downcast_ref::<&str>() { + (*value).to_owned() + } else { + "non-string panic".to_owned() + }; + results.push(AuditFileEvidence { + status: "unsupported".to_owned(), + file: name, + evidence: None, + error: Some(format!("panic: {message}")), + }); + unsupported += 1; + } + } + } + panic::set_hook(panic_hook); + return Ok(AuditEvidence { + evidence_version: 1, + oracle: ToolEvidence::current()?, + action: "audit".to_owned(), + file_count: results.len(), + supported_count: supported, + unsupported_count: unsupported, + files: results, + }); +} + +fn generate(args: &[String]) -> Result { + validate_options(args, &["--output", "--evidence"])?; + let directory = PathBuf::from(required(args, "--output")?); + let files = generate_all(&directory)?; + let mut evidence = Vec::with_capacity(files.len()); + for (case, version, path) in files { + let item = inspect_file(&path, case.id())?; + verify_owned(case, version, &item)?; + evidence.push(item); + } + return OracleEvidence::new("generate", evidence); +} + +fn inspect(args: &[String]) -> Result { + validate_options(args, &["--input", "--case-id", "--evidence"])?; + let path = PathBuf::from(required(args, "--input")?); + let case_id = required(args, "--case-id")?; + let evidence = inspect_file(&path, &case_id)?; + return OracleEvidence::new("inspect", vec![evidence]); +} + +fn diagnose_rule3_near_neighbor(args: &[String]) -> Result { + validate_options(args, &["--output", "--evidence"])?; + let directory = PathBuf::from(required(args, "--output")?); + let files = generate_list_rule3_near_neighbor(&directory)?; + let mut evidence = Vec::with_capacity(files.len()); + for (version, path) in files { + let item = inspect_file(&path, LIST_RULE3_NEAR_NEIGHBOR)?; + verify_list_rule3_near_neighbor(version, &item)?; + evidence.push(item); + } + return OracleEvidence::new("diagnose-rule3-near-neighbor", evidence); +} + +#[derive(serde::Deserialize)] +#[serde(untagged)] +enum ExpectedEvidence { + File(FileEvidence), + Report(OracleEvidence), +} + +fn read_expected(path: &Path, actual: &FileEvidence) -> Result { + let file = File::open(path)?; + let expected: ExpectedEvidence = serde_json::from_reader(file)?; + return match expected { + ExpectedEvidence::File(file) => Ok(file), + ExpectedEvidence::Report(report) => report + .files + .into_iter() + .find(|file| file.case_id == actual.case_id && file.file_name == actual.file_name) + .ok_or_else(|| "expected report has no matching file evidence".into()), + }; +} + +fn verify(args: &[String]) -> Result { + validate_options(args, &["--input", "--expected", "--case-id", "--evidence"])?; + let path = PathBuf::from(required(args, "--input")?); + let expected_path = PathBuf::from(required(args, "--expected")?); + let case_id = required(args, "--case-id")?; + let actual = inspect_file(&path, &case_id)?; + let expected = read_expected(&expected_path, &actual)?; + if actual != expected { + return Err(format!("oracle evidence differs from {}", expected_path.display()).into()); + } + return OracleEvidence::new("verify", vec![actual]); +} + +fn run() -> Result<()> { + let arguments: Vec = env::args().skip(1).collect(); + let (command, args) = arguments + .split_first() + .ok_or_else(|| -> DynError { usage().into() })?; + if command == "audit" { + let evidence = audit(args)?; + output_audit(&evidence, option(args, "--evidence")?)?; + return Ok(()); + } + let evidence = match command.as_str() { + "generate" => generate(args)?, + "diagnose-rule3-near-neighbor" => diagnose_rule3_near_neighbor(args)?, + "inspect" => inspect(args)?, + "verify" => verify(args)?, + "versions" => { + validate_options(args, &["--evidence"])?; + OracleEvidence::new("versions", vec![])? + } + _ => return Err(usage().into()), + }; + output_evidence(&evidence, option(args, "--evidence")?)?; + return Ok(()); +} + +fn main() { + if let Err(error) = run() { + eprintln!("arrow-rs-oracle: {error}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::io::ErrorKind; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use crate::evidence::inspect_file; + + struct TestDirectory(PathBuf); + + impl TestDirectory { + fn new() -> Result { + static NEXT: AtomicUsize = AtomicUsize::new(0); + for _ in 0..16 { + let id = NEXT.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "parquet-jl-n5-arrow-rs-{}-{id}", + std::process::id() + )); + match std::fs::create_dir(&path) { + Ok(()) => return Ok(Self(path)), + Err(error) if error.kind() == ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + } + } + return Err("cannot create a unique test directory".into()); + } + + fn path(&self) -> &Path { + return &self.0; + } + } + + impl Drop for TestDirectory { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[test] + fn owned_fixtures_are_deterministic_and_self_verifying() -> Result<()> { + let first = TestDirectory::new()?; + let first_files = generate_all(first.path())?; + assert_eq!(first_files.len(), 6); + let mut first_hashes = BTreeSet::new(); + for (case, version, path) in &first_files { + let file = inspect_file(path, case.id())?; + verify_owned(*case, *version, &file)?; + first_hashes.insert((file.file_name, file.sha256)); + } + assert_eq!(first_hashes.len(), 6); + let mut first_near_neighbor_hashes = BTreeSet::new(); + for (version, path) in generate_list_rule3_near_neighbor(first.path())? { + let file = inspect_file(&path, LIST_RULE3_NEAR_NEIGHBOR)?; + verify_list_rule3_near_neighbor(version, &file)?; + first_near_neighbor_hashes.insert((file.file_name, file.sha256)); + } + assert_eq!(first_near_neighbor_hashes.len(), 2); + + let second = TestDirectory::new()?; + let second_files = generate_all(second.path())?; + let mut second_hashes = BTreeSet::new(); + for (case, version, path) in &second_files { + let file = inspect_file(path, case.id())?; + verify_owned(*case, *version, &file)?; + second_hashes.insert((file.file_name, file.sha256)); + } + assert_eq!(first_hashes, second_hashes); + let mut second_near_neighbor_hashes = BTreeSet::new(); + for (version, path) in generate_list_rule3_near_neighbor(second.path())? { + let file = inspect_file(&path, LIST_RULE3_NEAR_NEIGHBOR)?; + verify_list_rule3_near_neighbor(version, &file)?; + second_near_neighbor_hashes.insert((file.file_name, file.sha256)); + } + assert_eq!(first_near_neighbor_hashes, second_near_neighbor_hashes); + return Ok(()); + } + + #[test] + fn audit_records_supported_and_unsupported_files() -> Result<()> { + let directory = TestDirectory::new()?; + let generated = generate_all(directory.path())?; + let retained = generated + .first() + .ok_or_else(|| -> DynError { "generated fixture list is empty".into() })? + .2 + .clone(); + for (_, _, path) in generated.iter().skip(1) { + fs::remove_file(path)?; + } + let invalid = directory.path().join("invalid.parquet"); + fs::write(&invalid, [0_u8, 1, 2, 3])?; + + let report = audit(&["--input".to_owned(), directory.path().display().to_string()])?; + assert_eq!(report.file_count, 2); + assert_eq!(report.supported_count, 1); + assert_eq!(report.unsupported_count, 1); + let invalid_result = report + .files + .iter() + .find(|file| file.file == "invalid.parquet") + .ok_or_else(|| -> DynError { "invalid audit result is absent".into() })?; + assert_eq!(invalid_result.status, "unsupported"); + assert!(invalid_result.evidence.is_none()); + assert!(invalid_result.error.is_some()); + let retained_name = retained.file_name().unwrap().to_string_lossy(); + let valid_result = report + .files + .iter() + .find(|file| file.file == retained_name) + .ok_or_else(|| -> DynError { "valid audit result is absent".into() })?; + assert_eq!(valid_result.status, "supported"); + assert!(valid_result.evidence.is_some()); + assert!(valid_result.error.is_none()); + return Ok(()); + } +} diff --git a/test/conformance/n5/oracles/image/validate-image.sh b/test/conformance/n5/oracles/image/validate-image.sh new file mode 100755 index 0000000..6007b63 --- /dev/null +++ b/test/conformance/n5/oracles/image/validate-image.sh @@ -0,0 +1,160 @@ +#!/bin/sh +set -eu + +fail() { + echo "validate-n5-oracle-image: $1" >&2 + exit 65 +} + +work=$(mktemp -d) +cleanup() { + rm -rf "$work" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +test "$(uname -s)" = "Linux" || fail "operating system is not Linux" +test "$(uname -m)" = "x86_64" || fail "architecture is not x86_64" +java_output=$(java -version 2>&1) || fail "Java version query failed" +case "$java_output" in + *'openjdk version "11.0.28"'*'Temurin-11.0.28+6 (build 11.0.28+6)'*) ;; + *) fail "Java is not exact Temurin 11.0.28+6" ;; +esac +maven_output=$(mvn --version 2>&1) || fail "Maven version query failed" +case "$maven_output" in + 'Apache Maven 3.9.8 '* | 'Apache Maven 3.9.8') ;; + *) fail "Maven is not version 3.9.8" ;; +esac +test "$(rustc --version)" = "rustc 1.96.1 (31fca3adb 2026-06-26)" || + fail "rustc is not the pinned build" +cargo_output=$(cargo --version 2>&1) || fail "Cargo version query failed" +test "$cargo_output" = "cargo 1.96.1 (356927216 2026-06-26)" || + fail "Cargo is not version 1.96.1" +test "$(dpkg-query -W -f='${Version}' perl)" = "5.38.2-3.2ubuntu0.3" || + fail "Perl package differs from the base-image pin" +test "$(perl -e 'print $^V')" = "v5.38.2" || + fail "Perl is not version 5.38.2" +test "$(perl -MJSON::PP -e 'print $JSON::PP::VERSION')" = "4.16" || + fail "JSON::PP is not version 4.16" +test "$(dpkg-query -W -f='${Version}' build-essential)" = "12.10ubuntu1" || + fail "build-essential differs from the snapshot pin" +test "$(dpkg-query -W -f='${Version}' ca-certificates)" = \ + "20260601~24.04.1" || fail "ca-certificates differs from the snapshot pin" +test "$(dpkg-query -W -f='${Version}' cmake)" = "3.28.3-1build7" || + fail "cmake differs from the snapshot pin" +test "$(dpkg-query -W -f='${Version}' git)" = "1:2.43.0-1ubuntu7.3" || + fail "git differs from the snapshot pin" +test "$(dpkg-query -W -f='${Version}' pkg-config)" = "1.8.1-2build1" || + fail "pkg-config differs from the snapshot pin" +test "$(dpkg-query -W -f='${Version}' xz-utils)" = \ + "5.6.1+really5.4.5-1ubuntu0.3" || + fail "xz-utils differs from the snapshot pin" + +printf '%s\n' \ + 'base_image=eclipse-temurin:11.0.28_6-jdk@sha256:ab2527b3c9b7c15bc88f60dec19b2aa39939a6e0045fb8f538eeecbd7af59c69' \ + 'source_date_epoch=1787356800' \ + 'ubuntu_snapshot=20260822T000000Z' \ + 'ubuntu_build_essential=12.10ubuntu1' \ + 'ubuntu_ca_certificates=20260601~24.04.1' \ + 'ubuntu_cmake=3.28.3-1build7' \ + 'ubuntu_git=1:2.43.0-1ubuntu7.3' \ + 'ubuntu_pkg_config=1.8.1-2build1' \ + 'ubuntu_xz_utils=5.6.1+really5.4.5-1ubuntu0.3' \ + 'maven_version=3.9.8' \ + 'maven_archive_sha512=7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2' \ + 'rust_version=1.96.1' \ + 'rust_channel_manifest_sha256=87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975' \ + 'rust_tarball_sha256=d29ccb1559a177c4e72291f6e5f629de7fe8885e7521ca47802627544b121e95' \ + 'perl_package_version=5.38.2-3.2ubuntu0.3' \ + 'perl_version=v5.38.2' \ + 'json_pp_version=4.16' \ + 'parquet_java_version=1.17.1' \ + 'parquet_java_commit=78a8d3230eb4769db93de5f2f2e18363c04cae81' \ + 'arrow_rs_version=59.2.0' \ + 'arrow_rs_commit=782e5a685501a9db6cc8e9a3b7cbff894940c47a' \ + > "$work/toolchains.txt" +cmp -s "$work/toolchains.txt" /opt/n5/manifests/toolchains.txt || + fail "toolchain manifest differs from the exact contract" +channel_output=$(sha256sum /opt/n5/pins/channel-rust-1.96.1.toml) || + fail "Rust channel pin is absent" +channel_hash=${channel_output%% *} +test "$channel_hash" = \ + "87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975" || + fail "Rust channel pin hash differs" + +printf '%s\n' \ + cargo-dependency-tree.txt \ + cargo-vendor.sha256 \ + harness-source.sha256 \ + maven-artifacts.sha256 \ + maven-dependency-tree.txt \ + toolchains.txt > "$work/expected-manifests" +find /opt/n5/manifests -mindepth 1 -printf '%P\n' > "$work/actual-manifests" || + fail "cannot enumerate image manifests" +LC_ALL=C sort "$work/actual-manifests" -o "$work/actual-manifests" +cmp -s "$work/expected-manifests" "$work/actual-manifests" || + fail "image manifest directory is not closed" + +special=$(find /opt/n5/maven/repository ! -type d ! -type f -print -quit) || + fail "cannot inspect the Maven repository" +test -z "$special" || fail "Maven repository contains a non-regular entry" +(cd /opt/n5/maven/repository && + find . -type f -print0 > "$work/maven-files") || + fail "cannot enumerate the Maven repository" +LC_ALL=C sort -z "$work/maven-files" -o "$work/maven-files" +(cd /opt/n5/maven/repository && + xargs -0 sha256sum < "$work/maven-files" > "$work/maven.sha256") || + fail "cannot hash the Maven repository" +cmp -s "$work/maven.sha256" /opt/n5/manifests/maven-artifacts.sha256 || + fail "Maven repository content is not exact" + +special=$(find /opt/n5/vendor /opt/n5/cargo /opt/n5/pins \ + ! -type d ! -type f -print -quit) || fail "cannot inspect Cargo content" +test -z "$special" || fail "Cargo content contains a non-regular entry" +(cd / && find opt/n5/vendor opt/n5/cargo opt/n5/pins -type f -print0 \ + > "$work/cargo-files") || fail "cannot enumerate Cargo content" +printf '%s\000' usr/local/bin/parquet-jl-n5-arrow-rs-oracle \ + >> "$work/cargo-files" +LC_ALL=C sort -z "$work/cargo-files" -o "$work/cargo-files" +(cd / && xargs -0 sha256sum < "$work/cargo-files" \ + > "$work/cargo.sha256") || fail "cannot hash Cargo content" +cmp -s "$work/cargo.sha256" /opt/n5/manifests/cargo-vendor.sha256 || + fail "Cargo content is not exact" + +special=$(find /opt/bootstrap ! -type d ! -type f -print -quit) || + fail "cannot inspect harness source" +test -z "$special" || fail "harness source contains a non-regular entry" +(cd / && find opt/bootstrap -type f -print0 > "$work/source-files") || + fail "cannot enumerate harness source" +printf '%s\000' opt/n5/manifests/toolchains.txt \ + usr/local/bin/validate-n5-oracle-image >> "$work/source-files" +LC_ALL=C sort -z "$work/source-files" -o "$work/source-files" +(cd / && xargs -0 sha256sum < "$work/source-files" \ + > "$work/source.sha256") || fail "cannot hash harness source" +cmp -s "$work/source.sha256" /opt/n5/manifests/harness-source.sha256 || + fail "harness source is not exact" + +(cd /opt/bootstrap/parquet-java && + mvn --batch-mode --no-transfer-progress --offline test package && + mvn --batch-mode --no-transfer-progress --offline dependency:tree \ + -DoutputFile="$work/maven-tree.txt" -DappendOutput=false) || + fail "offline Maven validation failed" +cmp -s "$work/maven-tree.txt" /opt/n5/manifests/maven-dependency-tree.txt || + fail "Maven dependency tree is not exact" +(cd /opt/bootstrap/arrow-rs && + cargo metadata --format-version=1 --offline --locked >/dev/null && + cargo tree --offline --locked > "$work/cargo-tree.txt") || + fail "offline Cargo validation failed" +cmp -s "$work/cargo-tree.txt" /opt/n5/manifests/cargo-dependency-tree.txt || + fail "Cargo dependency tree is not exact" +parquet-jl-n5-arrow-rs-oracle generate \ + --output "$work/golden" \ + --evidence "$work/evidence.json" || fail "Rust oracle self-check failed" + +cleanup +trap - EXIT HUP INT TERM +if [ "$#" -gt 0 ]; then + exec "$@" +fi diff --git a/test/conformance/n5/oracles/parquet-java/.gitignore b/test/conformance/n5/oracles/parquet-java/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/test/conformance/n5/oracles/parquet-java/README.md b/test/conformance/n5/oracles/parquet-java/README.md new file mode 100644 index 0000000..7362f59 --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/README.md @@ -0,0 +1,70 @@ +# Parquet Java N5 oracle + +This test-only project pins Parquet Java 1.17.1 from peeled upstream commit +`78a8d3230eb4769db93de5f2f2e18363c04cae81`. It follows that release's Hadoop +3.3.0 and SLF4J 1.7.33 pins. The test pin is JUnit 5.11.4. All Maven plugins have +fixed versions. The build requires Maven 3.9.8 and emits Java 11 bytecode. The +locked gate uses Temurin 11.0.28+6. Local checks can use a newer Java runtime. + +The Maven 3.9.8 archive SHA-512 is +`7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2`. +The locked N5 OCI image supplies the complete Maven repository for offline runs. + +The harness owns fifteen legacy nested fixtures and diagnostic controls. It writes each with +Parquet Java V1 and V2 pages. The writer uses uncompressed pages, disables the +dictionary, enables page checksums and validation, and fixes page and row-group +targets at 1 MiB and 128 MiB. The JSON evidence records this configuration. The +harness canonicalizes each writer-owned footer by stable Parquet encoding enum +value. This removes Parquet Java's process-dependent encoding-set iteration order. +The canonicalizer preserves duplicate encodings and rejects the file unless the +data/page prefix and every other decoded footer field stay exact. It also requires +the original and canonical footers to be exact Compact Thrift round trips. +harness then checks raw `Group` rows, physical schemas, row-group-local repetition +and definition levels, dense values, page versions, and page row counts. Supported logical cases +are also read through `AvroParquetReader` with +`parquet.avro.add-list-element-records=false`. Unsupported high-level layouts +remain explicit diagnostic rejections. + +The binding LIST rule-3 fixture retains the specification's `LIST` annotation on +the repeated inner group. Inferred Avro must return `LIST>` and all +rows exactly. A separately identified unannotated near-neighbor records Parquet +Java 1.17.1's diagnostic `ClassCastException`. No explicit Avro schema changes the +binding interpretation. + +The direct legacy LIST-of-MAP controls are also split. The INT32-key fixture +retains exact ordered physical pairs and records Avro's unsupported-key rejection. +The UTF8-key fixture retains duplicate physical pairs and requires inferred Avro +to report `array>` with last-value-wins logical rows. + +Generate and verify the Java-owned fixtures: + +```sh +./run.sh generate --output /tmp/n5-java --evidence /tmp/n5-java.jsonl +``` + +Verify Parquet files produced or rewritten by Julia. Verification is always +strict. Every input must bind through Java case metadata, a Java case ID in its +file name, or the explicit single-file case and page-version options: + +```sh +./run.sh verify --input /tmp/julia-files --evidence /tmp/julia-java.jsonl +./run.sh verify --input /tmp/julia-rule1.parquet \ + --case-id list_rule1_primitive --page-version v1 \ + --evidence /tmp/julia-rule1-java.jsonl +``` + +Inspect other files without applying a Java-owned expected case: + +```sh +./run.sh inspect --input /tmp/files --evidence /tmp/inspect.jsonl +``` + +Run the deterministic generation and semantic self-test: + +```sh +./check.sh +``` + +Pass `--offline` before the command for the locked OCI gate. Every command fails +when its input set is empty. Evidence is UTF-8 JSON Lines. It has no timestamps +or host paths. diff --git a/test/conformance/n5/oracles/parquet-java/check.sh b/test/conformance/n5/oracles/parquet-java/check.sh new file mode 100755 index 0000000..4c188c9 --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/check.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")" && pwd) +maven_args=(-q) +if [[ ${1:-} == "--offline" ]]; then + maven_args+=(--offline) + shift +fi +if [[ $# -ne 0 ]]; then + echo "usage: check.sh [--offline]" >&2 + exit 64 +fi + +(cd "$root" && mvn "${maven_args[@]}" test) diff --git a/test/conformance/n5/oracles/parquet-java/pom.xml b/test/conformance/n5/oracles/parquet-java/pom.xml new file mode 100644 index 0000000..58086ae --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/pom.xml @@ -0,0 +1,142 @@ + + + 4.0.0 + + org.julialang.parquet + parquet-java-n5-oracle + 1.0.0 + Parquet.jl N5 Parquet Java oracle + + + UTF-8 + UTF-8 + 2026-01-01T00:00:00Z + 11 + 1.17.1 + 78a8d3230eb4769db93de5f2f2e18363c04cae81 + 3.3.0 + 1.7.33 + 5.11.4 + + + + + org.apache.parquet + parquet-avro + ${parquet.version} + + + org.apache.hadoop + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + runtime + + + org.slf4j + slf4j-simple + ${slf4j.version} + runtime + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.4.0 + + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + -Dfile.encoding=UTF-8 + + false + en + US + UTC + warn + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + org.julialang.parquet.n5.OracleMain + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.8.1 + + + copy-runtime-dependencies + package + + copy-dependencies + + + runtime + ${project.build.directory}/dependency + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + pin-build-runtime + + enforce + + + + + [3.9.8,3.9.9) + + + [11,) + + + + + + + + + diff --git a/test/conformance/n5/oracles/parquet-java/run.sh b/test/conformance/n5/oracles/parquet-java/run.sh new file mode 100755 index 0000000..32884fd --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")" && pwd) +export LC_ALL=C +maven_args=(-q) +if [[ ${1:-} == "--offline" ]]; then + maven_args+=(--offline) + shift +fi +if [[ $# -eq 0 ]]; then + echo "usage: run.sh [--offline] [options]" >&2 + exit 64 +fi + +(cd "$root" && mvn "${maven_args[@]}" -DskipTests package) +classpath="$root/target/classes" +dependency_count=0 +for dependency in "$root"/target/dependency/*.jar; do + [[ -f $dependency ]] || continue + classpath="$classpath:$dependency" + dependency_count=$((dependency_count + 1)) +done +[[ $dependency_count -gt 0 ]] || { + echo "run.sh: Maven runtime dependencies are absent" >&2 + exit 65 +} +exec java \ + -Dfile.encoding=UTF-8 \ + -Duser.language=en \ + -Duser.country=US \ + -Duser.timezone=UTC \ + -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \ + -Dparquet.avro.add-list-element-records=false \ + -cp "$classpath" \ + org.julialang.parquet.n5.OracleMain "$@" diff --git a/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/CanonicalParquetFooter.java b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/CanonicalParquetFooter.java new file mode 100644 index 0000000..80db1fb --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/CanonicalParquetFooter.java @@ -0,0 +1,178 @@ +package org.julialang.parquet.n5; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import org.apache.parquet.format.ColumnChunk; +import org.apache.parquet.format.ColumnMetaData; +import org.apache.parquet.format.Encoding; +import org.apache.parquet.format.FileMetaData; +import org.apache.parquet.format.RowGroup; +import org.apache.parquet.format.Util; + +final class CanonicalParquetFooter { + private static final byte[] MAGIC = new byte[] {'P', 'A', 'R', '1'}; + private static final int TRAILER_SIZE = 8; + + private CanonicalParquetFooter() {} + + static void canonicalize(Path path) throws IOException { + byte[] originalFile = Files.readAllBytes(path); + Footer original = readFooter(originalFile); + FileMetaData canonicalMetadata = original.metadata.deepCopy(); + sortAllEncodings(canonicalMetadata); + assertOnlyEncodingOrderChanged(original.metadata, canonicalMetadata); + byte[] canonicalFooter = writeMetadata(canonicalMetadata); + if (canonicalFooter.length != original.bytes.length) { + throw new IOException("canonical footer length changed"); + } + byte[] canonicalFile = Arrays.copyOf(originalFile, originalFile.length); + System.arraycopy(canonicalFooter, 0, canonicalFile, original.start, canonicalFooter.length); + if (!Arrays.equals( + Arrays.copyOfRange(originalFile, 0, original.start), + Arrays.copyOfRange(canonicalFile, 0, original.start))) { + throw new IOException("canonicalization changed Parquet data or page bytes"); + } + Footer reparsed = readFooter(canonicalFile); + if (!canonicalMetadata.equals(reparsed.metadata)) { + throw new IOException("canonical footer changed during serialization"); + } + assertOnlyEncodingOrderChanged(original.metadata, reparsed.metadata); + if (Arrays.equals(originalFile, canonicalFile)) { + return; + } + replace(path, canonicalFile); + byte[] committed = Files.readAllBytes(path); + if (!Arrays.equals(canonicalFile, committed)) { + throw new IOException("committed canonical Parquet file changed"); + } + } + + static List sortedEncodings(List encodings) { + if (encodings == null) { + throw new IllegalArgumentException("missing encoding list"); + } + List result = new ArrayList<>(encodings); + result.sort(Comparator.comparingInt(Encoding::getValue)); + return result; + } + + private static Footer readFooter(byte[] file) throws IOException { + if (file.length < MAGIC.length + TRAILER_SIZE) { + throw new IOException("Parquet file is too short"); + } + requireMagic(file, 0); + requireMagic(file, file.length - MAGIC.length); + int footerLength = ByteBuffer.wrap(file, file.length - TRAILER_SIZE, Integer.BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .getInt(); + if (footerLength < 0 || footerLength > file.length - MAGIC.length - TRAILER_SIZE) { + throw new IOException("invalid Parquet footer length: " + footerLength); + } + int footerStart = file.length - TRAILER_SIZE - footerLength; + byte[] footer = Arrays.copyOfRange(file, footerStart, footerStart + footerLength); + ByteArrayInputStream input = new ByteArrayInputStream(footer); + FileMetaData metadata = Util.readFileMetaData(input); + if (input.available() != 0) { + throw new IOException("Parquet footer has trailing Thrift bytes"); + } + byte[] serialized = writeMetadata(metadata); + if (!Arrays.equals(footer, serialized)) { + throw new IOException("Parquet footer is not an exact Compact Thrift round trip"); + } + return new Footer(footerStart, footer, metadata); + } + + private static byte[] writeMetadata(FileMetaData metadata) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Util.writeFileMetaData(metadata, output); + return output.toByteArray(); + } + + private static void sortAllEncodings(FileMetaData metadata) throws IOException { + if (metadata.getRow_groups() == null) { + throw new IOException("Parquet footer has no row groups"); + } + for (RowGroup rowGroup : metadata.getRow_groups()) { + if (rowGroup.getColumns() == null) { + throw new IOException("Parquet row group has no columns"); + } + for (ColumnChunk column : rowGroup.getColumns()) { + ColumnMetaData columnMetadata = column.getMeta_data(); + if (columnMetadata == null || columnMetadata.getEncodings() == null) { + throw new IOException("Parquet column has no encoding metadata"); + } + List before = columnMetadata.getEncodings(); + List sorted = sortedEncodings(before); + if (before.size() != sorted.size()) { + throw new IOException("canonicalization changed encoding count"); + } + columnMetadata.setEncodings(sorted); + } + } + } + + private static void assertOnlyEncodingOrderChanged( + FileMetaData original, FileMetaData candidate) throws IOException { + FileMetaData expected = original.deepCopy(); + sortAllEncodings(expected); + if (!expected.equals(candidate)) { + throw new IOException("canonicalization changed a non-encoding footer field"); + } + } + + private static void replace(Path path, byte[] bytes) throws IOException { + Path absolute = path.toAbsolutePath().normalize(); + Path temporary = Files.createTempFile( + absolute.getParent(), "." + absolute.getFileName(), ".canonical"); + boolean committed = false; + try { + Files.write(temporary, bytes, StandardOpenOption.TRUNCATE_EXISTING); + try { + Files.move( + temporary, + absolute, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException error) { + Files.move(temporary, absolute, StandardCopyOption.REPLACE_EXISTING); + } + committed = true; + } finally { + if (!committed) { + Files.deleteIfExists(temporary); + } + } + } + + private static void requireMagic(byte[] file, int offset) throws IOException { + for (int index = 0; index < MAGIC.length; index++) { + if (file[offset + index] != MAGIC[index]) { + throw new IOException("invalid Parquet magic"); + } + } + } + + private static final class Footer { + final int start; + final byte[] bytes; + final FileMetaData metadata; + + Footer(int start, byte[] bytes, FileMetaData metadata) { + this.start = start; + this.bytes = bytes; + this.metadata = metadata; + } + } +} diff --git a/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/FixtureCases.java b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/FixtureCases.java new file mode 100644 index 0000000..815d008 --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/FixtureCases.java @@ -0,0 +1,809 @@ +package org.julialang.parquet.n5; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.avro.generic.GenericRecord; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; + +final class FixtureCases { + enum AvroMode { + SUCCESS, + REJECTED + } + + @FunctionalInterface + interface RowFactory { + List create(SimpleGroupFactory factory); + } + + @FunctionalInterface + interface AvroNormalizer { + Object normalize(GenericRecord row); + } + + static final class ExpectedColumn { + final String path; + final List repetition; + final List definition; + final List dense; + + ExpectedColumn(String path, List repetition, List definition, List dense) { + this.path = path; + this.repetition = repetition; + this.definition = definition; + this.dense = dense; + } + } + + static final class CaseSpec { + final String id; + final MessageType schema; + final RowFactory rows; + final List rawRows; + final List columns; + final AvroMode avroMode; + final List avroRows; + final AvroNormalizer avroNormalizer; + final String avroMaterializedSchema; + final String explicitAvroSchema; + final AvroMode explicitAvroMode; + final List explicitAvroRows; + + CaseSpec( + String id, + String schema, + RowFactory rows, + List rawRows, + List columns, + AvroMode avroMode, + List avroRows, + AvroNormalizer avroNormalizer) { + this( + id, + schema, + rows, + rawRows, + columns, + avroMode, + avroRows, + avroNormalizer, + null, + null, + null, + List.of()); + } + + CaseSpec( + String id, + String schema, + RowFactory rows, + List rawRows, + List columns, + AvroMode avroMode, + List avroRows, + AvroNormalizer avroNormalizer, + String avroMaterializedSchema) { + this( + id, + schema, + rows, + rawRows, + columns, + avroMode, + avroRows, + avroNormalizer, + avroMaterializedSchema, + null, + null, + List.of()); + } + + CaseSpec( + String id, + String schema, + RowFactory rows, + List rawRows, + List columns, + AvroMode avroMode, + List avroRows, + AvroNormalizer avroNormalizer, + String avroMaterializedSchema, + String explicitAvroSchema, + AvroMode explicitAvroMode, + List explicitAvroRows) { + this.id = id; + this.schema = MessageTypeParser.parseMessageType(schema); + this.rows = rows; + this.rawRows = rawRows; + this.columns = columns; + this.avroMode = avroMode; + this.avroRows = avroRows; + this.avroNormalizer = avroNormalizer; + this.avroMaterializedSchema = avroMaterializedSchema; + this.explicitAvroSchema = explicitAvroSchema; + this.explicitAvroMode = explicitAvroMode; + this.explicitAvroRows = explicitAvroRows; + } + + List createRows() { + return rows.create(new SimpleGroupFactory(schema)); + } + } + + private static final AvroNormalizer IDENTITY = row -> row; + private static final List CASES = buildCases(); + private static final Map BY_ID = indexCases(); + + private FixtureCases() {} + + static List all() { + return CASES; + } + + static CaseSpec find(String id) { + return BY_ID.get(id); + } + + private static Map indexCases() { + Map result = new LinkedHashMap<>(); + for (CaseSpec spec : CASES) { + if (result.put(spec.id, spec) != null) { + throw new IllegalStateException("duplicate case ID: " + spec.id); + } + } + return Collections.unmodifiableMap(result); + } + + private static List buildCases() { + List cases = new ArrayList<>(); + cases.add(rule1()); + cases.add(rule2()); + cases.add(rule3()); + cases.add(rule3UnannotatedDiagnostic()); + cases.add(rule4Array()); + cases.add(rule4Tuple()); + cases.add(rule5Required()); + cases.add(rule5OptionalPaired()); + cases.add(rule5OptionalExtended()); + cases.add(directListMap()); + cases.add(directListMapUtf8()); + cases.add(standardMap()); + cases.add(arbitraryMapNames()); + cases.add(standaloneMapKeyValue()); + cases.add(keyOnlyMap()); + return Collections.unmodifiableList(cases); + } + + private static CaseSpec rule1() { + String schema = "message list_rule1_primitive {\n" + + " optional group items (LIST) {\n" + + " repeated int32 element;\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_rule1_primitive", + schema, + FixtureCases::rule1Rows, + strings( + "G{items=null}", + "G{items=G{element=[]}}", + "G{items=G{element=[i32:10]}}", + "G{items=G{element=[i32:20,i32:30]}}"), + columns(column("items.element", ints(0, 0, 0, 0, 1), ints(0, 1, 2, 2, 2), + strings("10", "20", "30"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[10]}", + "{\"items\":[20,30]}"), + IDENTITY); + } + + private static CaseSpec rule2() { + String schema = "message list_rule2_struct {\n" + + " optional group items (LIST) {\n" + + " repeated group element {\n" + + " required int32 x;\n" + + " optional int32 y;\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_rule2_struct", + schema, + FixtureCases::rule2Rows, + strings( + "G{items=null}", + "G{items=G{element=[]}}", + "G{items=G{element=[G{x=i32:1,y=null}]}}", + "G{items=G{element=[G{x=i32:2,y=i32:20},G{x=i32:3,y=i32:30}]}}"), + columns( + column("items.element.x", ints(0, 0, 0, 0, 1), ints(0, 1, 2, 2, 2), + strings("1", "2", "3")), + column("items.element.y", ints(0, 0, 0, 0, 1), ints(0, 1, 2, 3, 3), + strings("20", "30"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[{\"x\":1,\"y\":null}]}", + "{\"items\":[{\"x\":2,\"y\":20},{\"x\":3,\"y\":30}]}"), + IDENTITY); + } + + private static CaseSpec rule3() { + String schema = "message list_rule3_nested {\n" + + " optional group items (LIST) {\n" + + " repeated group array (LIST) {\n" + + " repeated int32 array;\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_rule3_nested", + schema, + FixtureCases::rule3Rows, + strings( + "G{items=null}", + "G{items=G{array=[]}}", + "G{items=G{array=[G{array=[]}]}}", + "G{items=G{array=[G{array=[i32:1,i32:2]},G{array=[]},G{array=[i32:3]}]}}"), + columns(column("items.array.array", ints(0, 0, 0, 0, 2, 1, 1), + ints(0, 1, 2, 3, 3, 2, 3), strings("1", "2", "3"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[[]]}", + "{\"items\":[[1,2],[],[3]]}"), + IDENTITY, + "{\"type\":\"record\",\"name\":\"list_rule3_nested\",\"fields\":[" + + "{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\"," + + "\"items\":{\"type\":\"array\",\"items\":\"int\"}}],\"default\":null}]}"); + } + + private static CaseSpec rule3UnannotatedDiagnostic() { + String schema = "message list_rule3_unannotated_diagnostic {\n" + + " optional group items (LIST) {\n" + + " repeated group list {\n" + + " repeated int32 element;\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_rule3_unannotated_diagnostic", + schema, + FixtureCases::rule3UnannotatedRows, + strings( + "G{items=null}", + "G{items=G{list=[]}}", + "G{items=G{list=[G{element=[]}]}}", + "G{items=G{list=[G{element=[i32:1,i32:2]},G{element=[]},G{element=[i32:3]}]}}"), + columns(column("items.list.element", ints(0, 0, 0, 0, 2, 1, 1), + ints(0, 1, 2, 3, 3, 2, 3), strings("1", "2", "3"))), + AvroMode.REJECTED, + strings(), + IDENTITY); + } + + private static CaseSpec rule4Array() { + String schema = "message list_rule4_array {\n" + + " optional group items (LIST) {\n" + + " repeated group array {\n" + + " optional int32 value;\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_rule4_array", + schema, + factory -> rule4Rows(factory, "array", 4, true), + strings( + "G{items=null}", + "G{items=G{array=[]}}", + "G{items=G{array=[G{value=null}]}}", + "G{items=G{array=[G{value=i32:4},G{value=null}]}}"), + columns(column("items.array.value", ints(0, 0, 0, 0, 1), ints(0, 1, 2, 3, 2), + strings("4"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[{\"value\":null}]}", + "{\"items\":[{\"value\":4},{\"value\":null}]}"), + IDENTITY); + } + + private static CaseSpec rule4Tuple() { + String schema = "message list_rule4_tuple {\n" + + " optional group items (LIST) {\n" + + " repeated group items_tuple {\n" + + " optional int32 value;\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_rule4_tuple", + schema, + factory -> rule4Rows(factory, "items_tuple", 7, false), + strings( + "G{items=null}", + "G{items=G{items_tuple=[]}}", + "G{items=G{items_tuple=[G{value=i32:7}]}}", + "G{items=G{items_tuple=[G{value=null},G{value=i32:8}]}}"), + columns(column("items.items_tuple.value", ints(0, 0, 0, 0, 1), + ints(0, 1, 3, 2, 3), strings("7", "8"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[{\"value\":7}]}", + "{\"items\":[{\"value\":null},{\"value\":8}]}"), + IDENTITY); + } + + private static CaseSpec rule5Required() { + String schema = "message list_rule5_required {\n" + + " optional group items (LIST) {\n" + + " repeated group list {\n" + + " required int32 element;\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_rule5_required", + schema, + FixtureCases::rule5RequiredRows, + strings( + "G{items=null}", + "G{items=G{list=[]}}", + "G{items=G{list=[G{element=i32:10}]}}", + "G{items=G{list=[G{element=i32:20},G{element=i32:30}]}}"), + columns(column("items.list.element", ints(0, 0, 0, 0, 1), ints(0, 1, 2, 2, 2), + strings("10", "20", "30"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[10]}", + "{\"items\":[20,30]}"), + IDENTITY); + } + + private static CaseSpec rule5OptionalPaired() { + String schema = optionalRule5Schema("list_rule5_optional_paired"); + return new CaseSpec( + "list_rule5_optional_paired", + schema, + factory -> rule5OptionalRows(factory, 4, false), + strings( + "G{items=null}", + "G{items=G{list=[]}}", + "G{items=G{list=[G{element=null}]}}", + "G{items=G{list=[G{element=i32:4},G{element=null}]}}"), + columns(column("items.list.element", ints(0, 0, 0, 0, 1), ints(0, 1, 2, 3, 2), + strings("4"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[null]}", + "{\"items\":[4,null]}"), + IDENTITY); + } + + private static CaseSpec rule5OptionalExtended() { + String schema = optionalRule5Schema("list_rule5_optional_extended"); + return new CaseSpec( + "list_rule5_optional_extended", + schema, + factory -> rule5OptionalRows(factory, 5, true), + strings( + "G{items=null}", + "G{items=G{list=[]}}", + "G{items=G{list=[G{element=null}]}}", + "G{items=G{list=[G{element=i32:5},G{element=null},G{element=i32:6}]}}"), + columns(column("items.list.element", ints(0, 0, 0, 0, 1, 1), + ints(0, 1, 2, 3, 2, 3), strings("5", "6"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[null]}", + "{\"items\":[5,null,6]}"), + IDENTITY); + } + + private static CaseSpec directListMap() { + String schema = "message list_direct_map {\n" + + " optional group items (LIST) {\n" + + " repeated group map (MAP) {\n" + + " repeated group key_value {\n" + + " required int32 key;\n" + + " required int32 value;\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_direct_map", + schema, + FixtureCases::directListMapRows, + strings( + "G{items=null}", + "G{items=G{map=[]}}", + "G{items=G{map=[G{key_value=[]}]}}", + "G{items=G{map=[G{key_value=[G{key=i32:1,value=i32:10},G{key=i32:1,value=i32:20}]},G{key_value=[]},G{key_value=[G{key=i32:2,value=i32:30}]}]}}"), + columns( + column("items.map.key_value.key", ints(0, 0, 0, 0, 2, 1, 1), + ints(0, 1, 2, 3, 3, 2, 3), strings("1", "1", "2")), + column("items.map.key_value.value", ints(0, 0, 0, 0, 2, 1, 1), + ints(0, 1, 2, 3, 3, 2, 3), strings("10", "20", "30"))), + AvroMode.REJECTED, + strings(), + IDENTITY); + } + + private static CaseSpec directListMapUtf8() { + String schema = "message list_direct_map_utf8 {\n" + + " optional group items (LIST) {\n" + + " repeated group map (MAP) {\n" + + " repeated group key_value {\n" + + " required binary key (UTF8);\n" + + " required int32 value;\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "list_direct_map_utf8", + schema, + FixtureCases::directListMapUtf8Rows, + strings( + "G{items=null}", + "G{items=G{map=[]}}", + "G{items=G{map=[G{key_value=[]}]}}", + "G{items=G{map=[G{key_value=[G{key=utf8:a,value=i32:10}," + + "G{key=utf8:a,value=i32:20}]},G{key_value=[]}," + + "G{key_value=[G{key=utf8:b,value=i32:30}]}]}}"), + columns( + column("items.map.key_value.key", ints(0, 0, 0, 0, 2, 1, 1), + ints(0, 1, 2, 3, 3, 2, 3), strings("utf8:a", "utf8:a", "utf8:b")), + column("items.map.key_value.value", ints(0, 0, 0, 0, 2, 1, 1), + ints(0, 1, 2, 3, 3, 2, 3), strings("10", "20", "30"))), + AvroMode.SUCCESS, + strings( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[{}]}", + "{\"items\":[{\"a\":20},{},{\"b\":30}]}"), + IDENTITY, + "{\"type\":\"record\",\"name\":\"list_direct_map_utf8\",\"fields\":[" + + "{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\"," + + "\"items\":{\"type\":\"map\",\"values\":\"int\"}}],\"default\":null}]}"); + } + + private static CaseSpec standardMap() { + String schema = "message map_standard {\n" + + " optional group map (MAP) {\n" + + " repeated group key_value {\n" + + " required binary key (UTF8);\n" + + " optional int32 value;\n" + + " }\n" + + " }\n" + + "}"; + return mapCase("map_standard", schema, "map", "key_value", "key", "value"); + } + + private static CaseSpec arbitraryMapNames() { + String schema = "message map_arbitrary_names {\n" + + " optional group bag (MAP) {\n" + + " repeated group pairs {\n" + + " required binary left (UTF8);\n" + + " optional int32 right;\n" + + " }\n" + + " }\n" + + "}"; + return mapCase("map_arbitrary_names", schema, "bag", "pairs", "left", "right"); + } + + private static CaseSpec standaloneMapKeyValue() { + String schema = "message map_standalone_mkv {\n" + + " optional group map (MAP_KEY_VALUE) {\n" + + " repeated group entries (MAP_KEY_VALUE) {\n" + + " required binary key (UTF8);\n" + + " optional int32 value;\n" + + " }\n" + + " }\n" + + "}"; + return mapCase("map_standalone_mkv", schema, "map", "entries", "key", "value"); + } + + private static CaseSpec keyOnlyMap() { + String schema = "message map_key_only {\n" + + " required group map (MAP) {\n" + + " repeated group key_value {\n" + + " required binary key (UTF8);\n" + + " }\n" + + " }\n" + + "}"; + return new CaseSpec( + "map_key_only", + schema, + FixtureCases::keyOnlyMapRows, + strings( + "G{map=G{key_value=[]}}", + "G{map=G{key_value=[G{key=utf8:k1}]}}", + "G{map=G{key_value=[G{key=utf8:k2},G{key=utf8:k2}]}}"), + columns(column("map.key_value.key", ints(0, 0, 0, 1), ints(0, 1, 1, 1), + strings("utf8:k1", "utf8:k2", "utf8:k2"))), + AvroMode.REJECTED, + strings(), + IDENTITY); + } + + private static CaseSpec mapCase( + String id, String schema, String outer, String entry, String key, String value) { + String prefix = outer + "." + entry + "."; + String rawOuter = outer; + String rawEntry = entry; + String rawKey = key; + String rawValue = value; + List rawRows = strings( + "G{" + rawOuter + "=null}", + "G{" + rawOuter + "=G{" + rawEntry + "=[]}}", + "G{" + rawOuter + "=G{" + rawEntry + "=[G{" + rawKey + "=utf8:a," + rawValue + "=null}]}}", + "G{" + rawOuter + "=G{" + rawEntry + "=[G{" + rawKey + "=utf8:a," + rawValue + + "=i32:1},G{" + rawKey + "=utf8:a," + rawValue + "=i32:2},G{" + rawKey + + "=utf8:b," + rawValue + "=i32:3}]}}", + "G{" + rawOuter + "=G{" + rawEntry + "=[G{" + rawKey + "=utf8:c," + rawValue + "=i32:4}]}}"); + List avroRows = strings( + "{\"" + outer + "\":null}", + "{\"" + outer + "\":{}}", + "{\"" + outer + "\":{\"a\":null}}", + "{\"" + outer + "\":{\"a\":2,\"b\":3}}", + "{\"" + outer + "\":{\"c\":4}}"); + return new CaseSpec( + id, + schema, + factory -> mapRows(factory, outer, entry, key, value), + rawRows, + columns( + column(prefix + key, ints(0, 0, 0, 0, 1, 1, 0), ints(0, 1, 2, 2, 2, 2, 2), + strings("utf8:a", "utf8:a", "utf8:a", "utf8:b", "utf8:c")), + column(prefix + value, ints(0, 0, 0, 0, 1, 1, 0), ints(0, 1, 2, 3, 3, 3, 3), + strings("1", "2", "3", "4"))), + AvroMode.SUCCESS, + avroRows, + IDENTITY); + } + + private static String optionalRule5Schema(String messageName) { + return "message " + messageName + " {\n" + + " optional group items (LIST) {\n" + + " repeated group list {\n" + + " optional int32 element;\n" + + " }\n" + + " }\n" + + "}"; + } + + private static List rule1Rows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").append("element", 10); + rows.add(one); + Group two = factory.newGroup(); + two.addGroup("items").append("element", 20).append("element", 30); + rows.add(two); + return rows; + } + + private static List rule2Rows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").addGroup("element").append("x", 1); + rows.add(one); + Group two = factory.newGroup(); + Group items = two.addGroup("items"); + items.addGroup("element").append("x", 2).append("y", 20); + items.addGroup("element").append("x", 3).append("y", 30); + rows.add(two); + return rows; + } + + private static List rule3Rows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").addGroup("array"); + rows.add(one); + Group nested = factory.newGroup(); + Group items = nested.addGroup("items"); + items.addGroup("array").append("array", 1).append("array", 2); + items.addGroup("array"); + items.addGroup("array").append("array", 3); + rows.add(nested); + return rows; + } + + private static List rule3UnannotatedRows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").addGroup("list"); + rows.add(one); + Group nested = factory.newGroup(); + Group items = nested.addGroup("items"); + items.addGroup("list").append("element", 1).append("element", 2); + items.addGroup("list"); + items.addGroup("list").append("element", 3); + rows.add(nested); + return rows; + } + + private static List rule4Rows( + SimpleGroupFactory factory, String wrapper, int firstValue, boolean firstNull) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + Group first = one.addGroup("items").addGroup(wrapper); + if (!firstNull) { + first.append("value", firstValue); + } + rows.add(one); + Group two = factory.newGroup(); + Group items = two.addGroup("items"); + if (firstNull) { + items.addGroup(wrapper).append("value", firstValue); + items.addGroup(wrapper); + } else { + items.addGroup(wrapper); + items.addGroup(wrapper).append("value", 8); + } + rows.add(two); + return rows; + } + + private static List rule5RequiredRows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").addGroup("list").append("element", 10); + rows.add(one); + Group two = factory.newGroup(); + Group items = two.addGroup("items"); + items.addGroup("list").append("element", 20); + items.addGroup("list").append("element", 30); + rows.add(two); + return rows; + } + + private static List rule5OptionalRows( + SimpleGroupFactory factory, int firstValue, boolean extended) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").addGroup("list"); + rows.add(one); + Group two = factory.newGroup(); + Group items = two.addGroup("items"); + items.addGroup("list").append("element", firstValue); + items.addGroup("list"); + if (extended) { + items.addGroup("list").append("element", 6); + } + rows.add(two); + return rows; + } + + private static List directListMapRows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").addGroup("map"); + rows.add(one); + Group nested = factory.newGroup(); + Group items = nested.addGroup("items"); + Group first = items.addGroup("map"); + first.addGroup("key_value").append("key", 1).append("value", 10); + first.addGroup("key_value").append("key", 1).append("value", 20); + items.addGroup("map"); + items.addGroup("map").addGroup("key_value").append("key", 2).append("value", 30); + rows.add(nested); + return rows; + } + + private static List directListMapUtf8Rows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, "items")); + Group one = factory.newGroup(); + one.addGroup("items").addGroup("map"); + rows.add(one); + Group nested = factory.newGroup(); + Group items = nested.addGroup("items"); + Group first = items.addGroup("map"); + first.addGroup("key_value").append("key", "a").append("value", 10); + first.addGroup("key_value").append("key", "a").append("value", 20); + items.addGroup("map"); + items.addGroup("map").addGroup("key_value").append("key", "b").append("value", 30); + rows.add(nested); + return rows; + } + + private static List mapRows( + SimpleGroupFactory factory, String outer, String entry, String key, String value) { + List rows = new ArrayList<>(); + rows.add(factory.newGroup()); + rows.add(presentEmpty(factory, outer)); + Group nullValue = factory.newGroup(); + nullValue.addGroup(outer).addGroup(entry).append(key, "a"); + rows.add(nullValue); + Group duplicate = factory.newGroup(); + Group duplicateMap = duplicate.addGroup(outer); + duplicateMap.addGroup(entry).append(key, "a").append(value, 1); + duplicateMap.addGroup(entry).append(key, "a").append(value, 2); + duplicateMap.addGroup(entry).append(key, "b").append(value, 3); + rows.add(duplicate); + Group last = factory.newGroup(); + last.addGroup(outer).addGroup(entry).append(key, "c").append(value, 4); + rows.add(last); + return rows; + } + + private static List keyOnlyMapRows(SimpleGroupFactory factory) { + List rows = new ArrayList<>(); + rows.add(presentEmpty(factory, "map")); + Group one = factory.newGroup(); + one.addGroup("map").addGroup("key_value").append("key", "k1"); + rows.add(one); + Group duplicate = factory.newGroup(); + Group map = duplicate.addGroup("map"); + map.addGroup("key_value").append("key", "k2"); + map.addGroup("key_value").append("key", "k2"); + rows.add(duplicate); + return rows; + } + + private static Group presentEmpty(SimpleGroupFactory factory, String field) { + Group root = factory.newGroup(); + root.addGroup(field); + return root; + } + + private static ExpectedColumn column( + String path, List repetition, List definition, List dense) { + return new ExpectedColumn(path, repetition, definition, dense); + } + + @SafeVarargs + private static List columns(ExpectedColumn... columns) { + return List.of(columns); + } + + private static List ints(Integer... values) { + return List.of(values); + } + + private static List strings(String... values) { + return List.of(values); + } +} diff --git a/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/Json.java b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/Json.java new file mode 100644 index 0000000..d69a197 --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/Json.java @@ -0,0 +1,164 @@ +package org.julialang.parquet.n5; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +final class Json { + private Json() {} + + static Map object(Object... pairs) { + if ((pairs.length & 1) != 0) { + throw new IllegalArgumentException("JSON object needs key/value pairs"); + } + Map result = new LinkedHashMap<>(); + for (int index = 0; index < pairs.length; index += 2) { + result.put((String) pairs[index], pairs[index + 1]); + } + return result; + } + + static String encode(Object value) { + StringBuilder output = new StringBuilder(); + append(output, value); + return output.toString(); + } + + static void writeLines(Path output, List> records) throws IOException { + Path target = output.toAbsolutePath().normalize(); + Path parent = target.getParent(); + if (parent == null) { + throw new IllegalArgumentException("evidence path has no parent: " + output); + } + Files.createDirectories(parent); + Path temporary = Files.createTempFile(parent, ".n5-java-evidence-", ".tmp"); + boolean committed = false; + try { + StringBuilder lines = new StringBuilder(); + for (Map record : records) { + lines.append(encode(record)).append('\n'); + } + Files.writeString(temporary, lines, StandardCharsets.UTF_8); + moveReplace(temporary, target); + committed = true; + } finally { + if (!committed) { + Files.deleteIfExists(temporary); + } + } + } + + static void moveReplace(Path source, Path target) throws IOException { + try { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException error) { + Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static void append(StringBuilder output, Object value) { + if (value == null) { + output.append("null"); + } else if (value instanceof String) { + appendString(output, (String) value); + } else if (value instanceof Boolean || value instanceof Byte || value instanceof Short + || value instanceof Integer || value instanceof Long) { + output.append(value); + } else if (value instanceof Float) { + appendFinite(output, ((Float) value).doubleValue()); + } else if (value instanceof Double) { + appendFinite(output, (Double) value); + } else if (value instanceof Map) { + appendMap(output, (Map) value); + } else if (value instanceof Collection) { + appendCollection(output, (Collection) value); + } else if (value.getClass().isArray()) { + throw new IllegalArgumentException("convert arrays to ordered collections before JSON encoding"); + } else { + throw new IllegalArgumentException("unsupported JSON value: " + value.getClass().getName()); + } + } + + private static void appendFinite(StringBuilder output, double value) { + if (!Double.isFinite(value)) { + appendString(output, Double.toString(value)); + return; + } + output.append(Double.toString(value)); + } + + private static void appendMap(StringBuilder output, Map value) { + output.append('{'); + boolean first = true; + for (Map.Entry entry : value.entrySet()) { + if (!(entry.getKey() instanceof String)) { + throw new IllegalArgumentException("JSON object key is not a string"); + } + if (!first) { + output.append(','); + } + first = false; + appendString(output, (String) entry.getKey()); + output.append(':'); + append(output, entry.getValue()); + } + output.append('}'); + } + + private static void appendCollection(StringBuilder output, Collection value) { + output.append('['); + boolean first = true; + for (Object element : value) { + if (!first) { + output.append(','); + } + first = false; + append(output, element); + } + output.append(']'); + } + + private static void appendString(StringBuilder output, String value) { + output.append('"'); + for (int index = 0; index < value.length(); index++) { + char character = value.charAt(index); + switch (character) { + case '"': + output.append("\\\""); + break; + case '\\': + output.append("\\\\"); + break; + case '\b': + output.append("\\b"); + break; + case '\f': + output.append("\\f"); + break; + case '\n': + output.append("\\n"); + break; + case '\r': + output.append("\\r"); + break; + case '\t': + output.append("\\t"); + break; + default: + if (character < 0x20 || Character.isSurrogate(character)) { + output.append(String.format("\\u%04x", (int) character)); + } else { + output.append(character); + } + } + } + output.append('"'); + } +} diff --git a/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/OracleMain.java b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/OracleMain.java new file mode 100644 index 0000000..772b12b --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/OracleMain.java @@ -0,0 +1,660 @@ +package org.julialang.parquet.n5; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.LocalInputFile; +import org.apache.parquet.io.LocalOutputFile; + +public final class OracleMain { + static final String PARQUET_JAVA_VERSION = "1.17.1"; + static final String PARQUET_JAVA_COMMIT = "78a8d3230eb4769db93de5f2f2e18363c04cae81"; + static final String CASE_METADATA_KEY = "parquet.jl.n5.case_id"; + static final String PAGE_VERSION_METADATA_KEY = "parquet.jl.n5.page_version"; + private static final String PRODUCER_METADATA_KEY = "parquet.jl.n5.producer"; + private static final String COMMIT_METADATA_KEY = "parquet.jl.n5.parquet_java_commit"; + private static final String AVRO_PROPERTY = "parquet.avro.add-list-element-records"; + private static final int PAGE_SIZE = 1024 * 1024; + private static final long ROW_GROUP_SIZE = 128L * 1024L * 1024L; + + private enum PageVersion { + V1("v1", ParquetProperties.WriterVersion.PARQUET_1_0, "DATA_PAGE_V1"), + V2("v2", ParquetProperties.WriterVersion.PARQUET_2_0, "DATA_PAGE_V2"); + + final String id; + final ParquetProperties.WriterVersion writerVersion; + final String pageType; + + PageVersion(String id, ParquetProperties.WriterVersion writerVersion, String pageType) { + this.id = id; + this.writerVersion = writerVersion; + this.pageType = pageType; + } + + static PageVersion parse(String value) { + for (PageVersion version : values()) { + if (version.id.equals(value)) { + return version; + } + } + return null; + } + } + + private static final class GeneratedFile { + final Path staged; + final Path target; + final ParquetEvidence.FileEvidence evidence; + + GeneratedFile(Path staged, Path target, ParquetEvidence.FileEvidence evidence) { + this.staged = staged; + this.target = target; + this.evidence = evidence; + } + } + + private OracleMain() {} + + public static void main(String[] arguments) { + try { + requireAvroConfiguration(); + run(arguments); + } catch (Exception error) { + System.err.println(Json.encode(Json.object( + "record", "error", + "error_class", error.getClass().getName(), + "message", error.getMessage()))); + error.printStackTrace(System.err); + System.exit(1); + } + } + + private static void run(String[] arguments) throws IOException { + if (arguments.length == 0) { + throw new IllegalArgumentException( + "usage: [options]; " + + "verify accepts --case-id ID --page-version v1|v2 for one file"); + } + String command = arguments[0]; + Options options = Options.parse(arguments, 1); + switch (command) { + case "generate": + runGenerate(options); + return; + case "verify": + runInspect(options, true); + return; + case "inspect": + runInspect(options, false); + return; + case "audit": + runAudit(options); + return; + case "self-test": + runSelfTest(options); + return; + default: + throw new IllegalArgumentException("unknown command: " + command); + } + } + + private static void runGenerate(Options options) throws IOException { + Path output = options.requiredPath("--output"); + Path evidencePath = options.requiredPath("--evidence"); + options.requireNoUnused(); + List files = generateFixtures(output); + Json.writeLines(evidencePath, evidenceRecords("generate", files)); + printSummary("generate", files.size()); + } + + private static void runInspect(Options options, boolean validateKnown) throws IOException { + Path input = options.requiredPath("--input"); + Path evidencePath = options.requiredPath("--evidence"); + boolean requireOwned = options.flag("--require-owned"); + String explicitCaseId = options.optional("--case-id"); + String explicitPageVersion = options.optional("--page-version"); + options.requireNoUnused(); + if (!validateKnown && (requireOwned || explicitCaseId != null || explicitPageVersion != null)) { + throw new IllegalArgumentException( + "--require-owned, --case-id, and --page-version are valid only for verify"); + } + List evidence = + inspectFiles(input, validateKnown, explicitCaseId, explicitPageVersion); + Json.writeLines(evidencePath, evidenceRecords(validateKnown ? "verify" : "inspect", evidence)); + printSummary(validateKnown ? "verify" : "inspect", evidence.size()); + } + + private static void runAudit(Options options) throws IOException { + Path input = options.requiredPath("--input"); + Path evidencePath = options.requiredPath("--evidence"); + options.requireNoUnused(); + List> records = auditRecords(input); + Json.writeLines(evidencePath, records); + int unsupported = 0; + for (Map record : records) { + if ("unsupported".equals(record.get("record"))) { + unsupported++; + } + } + printAuditSummary(records.size() - 1, unsupported); + } + + static List> auditRecords(Path input) throws IOException { + List sources = parquetFiles(input); + List> result = runRecord("audit", sources.size()); + int supported = 0; + int unsupported = 0; + for (Path source : sources) { + String relative = relativeName(input, source); + try { + ParquetEvidence.FileEvidence file = + ParquetEvidence.inspect(source, relative, row -> row, null); + result.add(file.toMap()); + supported++; + } catch (IOException | RuntimeException error) { + Throwable cause = rootCause(error); + result.add(Json.object( + "record", "unsupported", + "file", relative, + "error_class", cause.getClass().getName(), + "error_message", sanitizeError(cause.getMessage(), source))); + unsupported++; + } + } + Map run = result.get(0); + run.put("supported_count", supported); + run.put("unsupported_count", unsupported); + return result; + } + + static List verifyFiles( + Path input, String explicitCaseId, String explicitPageVersion) throws IOException { + return inspectFiles(input, true, explicitCaseId, explicitPageVersion); + } + + private static List inspectFiles( + Path input, boolean validateKnown, String explicitCaseId, String explicitPageVersion) + throws IOException { + List sources = parquetFiles(input); + if ((explicitCaseId == null) != (explicitPageVersion == null)) { + throw new IllegalArgumentException("--case-id and --page-version must be used together"); + } + FixtureCases.CaseSpec explicitSpec = null; + PageVersion explicitVersion = null; + if (explicitCaseId != null) { + if (sources.size() != 1) { + throw new IllegalArgumentException("explicit case binding requires one input file"); + } + explicitSpec = FixtureCases.find(explicitCaseId); + if (explicitSpec == null) { + throw new IllegalArgumentException("unknown Java case ID: " + explicitCaseId); + } + explicitVersion = PageVersion.parse(explicitPageVersion); + if (explicitVersion == null) { + throw new IllegalArgumentException("unknown page version: " + explicitPageVersion); + } + } + List evidence = new ArrayList<>(); + for (Path source : sources) { + String relative = relativeName(input, source); + FixtureCases.CaseSpec spec = explicitSpec == null ? resolveCase(source, relative) : explicitSpec; + if (validateKnown && spec == null) { + throw new IllegalStateException("no Java-owned case binding for " + relative); + } + FixtureCases.AvroNormalizer normalizer = spec == null ? row -> row : spec.avroNormalizer; + String explicitAvroSchema = spec == null ? null : spec.explicitAvroSchema; + ParquetEvidence.FileEvidence file = + ParquetEvidence.inspect(source, relative, normalizer, explicitAvroSchema); + if (validateKnown) { + PageVersion pageVersion = explicitVersion == null + ? resolvePageVersion(file, relative) : explicitVersion; + if (pageVersion == null) { + throw new IllegalStateException("no V1/V2 binding for " + relative); + } + validate(spec, pageVersion, file, false); + } + evidence.add(file); + } + return evidence; + } + + private static void runSelfTest(Options options) throws IOException { + Path work = options.requiredPath("--work"); + Path evidencePath = options.requiredPath("--evidence"); + options.requireNoUnused(); + List files = selfTest(work); + Json.writeLines(evidencePath, evidenceRecords("self-test", files)); + printSummary("self-test", files.size()); + } + + static List generateFixtures(Path output) throws IOException { + Files.createDirectories(output); + Path staging = Files.createTempDirectory(output, ".parquet-java-stage-"); + List generated = new ArrayList<>(); + boolean committed = false; + try { + for (FixtureCases.CaseSpec spec : FixtureCases.all()) { + for (PageVersion pageVersion : PageVersion.values()) { + String name = spec.id + "." + pageVersion.id + ".parquet"; + Path staged = staging.resolve(name); + writeFixture(staged, spec, pageVersion); + ParquetEvidence.FileEvidence evidence = + ParquetEvidence.inspect( + staged, name, spec.avroNormalizer, spec.explicitAvroSchema); + validateGeneratedIdentity(evidence); + validate(spec, pageVersion, evidence, true); + generated.add(new GeneratedFile(staged, output.resolve(name), evidence)); + } + } + for (GeneratedFile file : generated) { + try { + Files.move( + file.staged, + file.target, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (java.nio.file.AtomicMoveNotSupportedException error) { + Files.move(file.staged, file.target, StandardCopyOption.REPLACE_EXISTING); + } + } + committed = true; + return generated.stream().map(file -> file.evidence).collect(Collectors.toList()); + } finally { + deleteTree(staging); + if (!committed) { + for (GeneratedFile file : generated) { + Files.deleteIfExists(file.staged); + } + } + } + } + + static List selfTest(Path work) throws IOException { + Files.createDirectories(work); + Path first = work.resolve("first"); + Path second = work.resolve("second"); + List firstEvidence = generateFixtures(first); + List secondEvidence = generateFixtures(second); + Map firstHashes = hashes(firstEvidence); + Map secondHashes = hashes(secondEvidence); + if (!firstHashes.equals(secondHashes)) { + throw new IllegalStateException("repeated Java fixture generation changed file hashes"); + } + return firstEvidence; + } + + private static Map hashes(List evidence) { + Map result = new LinkedHashMap<>(); + for (ParquetEvidence.FileEvidence file : evidence) { + result.put(file.file, file.sha256); + } + return result; + } + + private static void writeFixture( + Path output, FixtureCases.CaseSpec spec, PageVersion pageVersion) throws IOException { + Map metadata = new LinkedHashMap<>(); + metadata.put(CASE_METADATA_KEY, spec.id); + metadata.put(PAGE_VERSION_METADATA_KEY, pageVersion.id); + metadata.put(PRODUCER_METADATA_KEY, "parquet-java"); + metadata.put(COMMIT_METADATA_KEY, PARQUET_JAVA_COMMIT); + try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(output)) + .withType(spec.schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .withRowGroupSize(ROW_GROUP_SIZE) + .withPageSize(PAGE_SIZE) + .withDictionaryEncoding(false) + .withValidation(true) + .withWriterVersion(pageVersion.writerVersion) + .withPageWriteChecksumEnabled(true) + .withExtraMetaData(metadata) + .build()) { + for (Group row : spec.createRows()) { + writer.write(row); + } + } + CanonicalParquetFooter.canonicalize(output); + CanonicalParquetFooter.canonicalize(output); + } + + private static void validate( + FixtureCases.CaseSpec spec, + PageVersion pageVersion, + ParquetEvidence.FileEvidence evidence, + boolean requireMetadataIdentity) { + if (requireMetadataIdentity || evidence.caseId() != null) { + requireEqual(spec.id, evidence.caseId(), "case metadata"); + } + if (pageVersion != null) { + if (requireMetadataIdentity || evidence.pageVersion() != null) { + requireEqual(pageVersion.id, evidence.pageVersion(), "page-version metadata"); + } + } + requireEqual(spec.schema, evidence.schema, "physical schema"); + requireEqual((long) spec.rawRows.size(), evidence.rowCount, "row count"); + requireEqual(spec.rawRows, evidence.rawRows, "raw Group rows"); + requireEqual(spec.columns.size(), evidence.columns.size(), "physical column count"); + for (FixtureCases.ExpectedColumn expected : spec.columns) { + ParquetEvidence.ColumnEvidence column = evidence.column(expected.path); + if (column == null) { + throw new IllegalStateException("missing column " + expected.path + " for " + spec.id); + } + requireEqual(evidence.rowGroups.size(), column.rowGroups.size(), + expected.path + " row-group count"); + if (requireMetadataIdentity) { + requireEqual(1, column.rowGroups.size(), expected.path + " generated row-group count"); + } + requireEqual(expected.repetition, column.repetition, expected.path + " repetition"); + requireEqual(expected.definition, column.definition, expected.path + " definition"); + requireEqual(expected.dense, column.dense, expected.path + " dense values"); + if (!column.dictionaries.isEmpty()) { + throw new IllegalStateException("unexpected dictionary page for " + expected.path); + } + if (column.pages.isEmpty()) { + throw new IllegalStateException("no data page for " + expected.path); + } + long groupRows = 0; + for (int groupIndex = 0; groupIndex < column.rowGroups.size(); groupIndex++) { + ParquetEvidence.ColumnRowGroupEvidence group = column.rowGroups.get(groupIndex); + requireEqual(groupIndex, group.rowGroup, expected.path + " row-group ordinal"); + groupRows = Math.addExact(groupRows, group.rows); + if (!group.dictionaries.isEmpty()) { + throw new IllegalStateException( + "unexpected row-group dictionary page for " + expected.path); + } + if (group.pages.isEmpty()) { + throw new IllegalStateException("no row-group data page for " + expected.path); + } + for (Map page : group.pages) { + requireEqual(pageVersion.pageType, page.get("type"), expected.path + " page type"); + } + } + requireEqual((long) spec.rawRows.size(), groupRows, expected.path + " row-group rows"); + if (column.rowGroups.size() == 1) { + ParquetEvidence.ColumnRowGroupEvidence group = column.rowGroups.get(0); + requireEqual(expected.repetition, group.repetition, expected.path + " row-group repetition"); + requireEqual(expected.definition, group.definition, expected.path + " row-group definition"); + requireEqual(expected.dense, group.dense, expected.path + " row-group dense values"); + } + } + validateAvro( + spec.id + " inferred Avro", + spec.avroMode, + spec.avroRows, + spec.avroMaterializedSchema, + evidence.avro); + if (spec.explicitAvroSchema == null) { + requireEqual(null, evidence.explicitAvro, "unexpected explicit Avro attempt"); + } else { + if (evidence.explicitAvro == null) { + throw new IllegalStateException("missing explicit Avro attempt for " + spec.id); + } + validateAvro( + spec.id + " explicit Avro", + spec.explicitAvroMode, + spec.explicitAvroRows, + null, + evidence.explicitAvro); + } + } + + private static void validateGeneratedIdentity(ParquetEvidence.FileEvidence evidence) { + requireEqual( + "parquet-mr version " + PARQUET_JAVA_VERSION + " (build " + PARQUET_JAVA_COMMIT + ")", + evidence.createdBy, + "generated created-by identity"); + requireEqual( + "parquet-java", evidence.metadata.get(PRODUCER_METADATA_KEY), "producer metadata"); + requireEqual( + PARQUET_JAVA_COMMIT, + evidence.metadata.get(COMMIT_METADATA_KEY), + "Parquet Java commit metadata"); + } + + private static void validateAvro( + String label, + FixtureCases.AvroMode mode, + List expectedRows, + String expectedSchema, + ParquetEvidence.AvroEvidence evidence) { + if (mode == FixtureCases.AvroMode.SUCCESS) { + if (!"success".equals(evidence.status)) { + throw new IllegalStateException( + label + " rejected with " + evidence.errorClass + ": " + evidence.errorMessage); + } + if (expectedSchema != null) { + requireEqual(expectedSchema, evidence.materializedSchema, label + " materialized schema"); + } + requireEqual(expectedRows, evidence.normalizedRows, label + " normalized rows"); + return; + } + requireEqual("rejected", evidence.status, label + " diagnostic rejection"); + if (evidence.errorClass == null || evidence.errorStack.isEmpty()) { + throw new IllegalStateException(label + " rejection has incomplete exception evidence"); + } + } + + private static void requireEqual(Object expected, Object actual, String label) { + if (!Objects.equals(expected, actual)) { + throw new IllegalStateException( + label + " mismatch: expected " + expected + ", got " + actual); + } + } + + private static FixtureCases.CaseSpec resolveCase(Path source, String relative) throws IOException { + try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(source))) { + String id = reader.getFooter().getFileMetaData().getKeyValueMetaData().get(CASE_METADATA_KEY); + if (id != null) { + return FixtureCases.find(id); + } + } + FixtureCases.CaseSpec result = null; + for (FixtureCases.CaseSpec spec : FixtureCases.all()) { + if (relative.contains(spec.id) + && (result == null || spec.id.length() > result.id.length())) { + result = spec; + } + } + return result; + } + + private static PageVersion resolvePageVersion( + ParquetEvidence.FileEvidence evidence, String relative) { + PageVersion version = PageVersion.parse(evidence.pageVersion()); + if (version != null) { + return version; + } + if (relative.contains(".v1.")) { + return PageVersion.V1; + } + if (relative.contains(".v2.")) { + return PageVersion.V2; + } + return null; + } + + private static List parquetFiles(Path input) throws IOException { + Path absolute = input.toAbsolutePath().normalize(); + List result = new ArrayList<>(); + if (Files.isRegularFile(absolute)) { + if (!absolute.getFileName().toString().endsWith(".parquet")) { + throw new IllegalArgumentException("input file is not .parquet: " + input); + } + result.add(absolute); + } else if (Files.isDirectory(absolute)) { + try (Stream paths = Files.walk(absolute)) { + paths.filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".parquet")) + .sorted(Comparator.comparing(path -> relativeName(absolute, path))) + .forEach(result::add); + } + } else { + throw new IllegalArgumentException("input does not exist: " + input); + } + if (result.isEmpty()) { + throw new IllegalStateException("input contains no .parquet files: " + input); + } + return result; + } + + private static String relativeName(Path input, Path source) { + Path absolute = input.toAbsolutePath().normalize(); + if (Files.isRegularFile(absolute)) { + return source.getFileName().toString(); + } + return absolute.relativize(source.toAbsolutePath().normalize()).toString() + .replace(File.separatorChar, '/'); + } + + private static List> evidenceRecords( + String command, List files) { + List> result = runRecord(command, files.size()); + for (ParquetEvidence.FileEvidence file : files) { + result.add(file.toMap()); + } + return result; + } + + private static List> runRecord(String command, int fileCount) { + List> result = new ArrayList<>(); + result.add(Json.object( + "record", "run", + "schema_version", 1, + "oracle", "parquet-java", + "command", command, + "parquet_java_version", PARQUET_JAVA_VERSION, + "parquet_java_commit", PARQUET_JAVA_COMMIT, + "avro_add_list_element_records", false, + "fixture_writer", Json.object( + "compression", CompressionCodecName.UNCOMPRESSED.name(), + "dictionary", false, + "page_size", PAGE_SIZE, + "row_group_size", ROW_GROUP_SIZE, + "page_checksums", true, + "validation", true, + "page_versions", List.of("v1", "v2")), + "java_version", System.getProperty("java.version"), + "java_vendor", System.getProperty("java.vendor"), + "file_count", fileCount)); + return result; + } + + private static Throwable rootCause(Throwable error) { + Throwable current = error; + while (current.getCause() != null && current.getCause() != current) { + current = current.getCause(); + } + return current; + } + + private static String sanitizeError(String value, Path source) { + if (value == null) { + return null; + } + return value.replace(source.toAbsolutePath().normalize().toString(), "") + .replace('\n', ' ').replace('\r', ' '); + } + + private static void printSummary(String command, int fileCount) { + System.out.println(Json.encode(Json.object( + "record", "summary", + "status", "ok", + "oracle", "parquet-java", + "command", command, + "file_count", fileCount))); + } + + private static void printAuditSummary(int fileCount, int unsupportedCount) { + System.out.println(Json.encode(Json.object( + "record", "summary", + "status", "ok", + "oracle", "parquet-java", + "command", "audit", + "file_count", fileCount, + "supported_count", fileCount - unsupportedCount, + "unsupported_count", unsupportedCount))); + } + + private static void requireAvroConfiguration() { + String value = System.getProperty(AVRO_PROPERTY); + if (!"false".equals(value)) { + throw new IllegalStateException( + AVRO_PROPERTY + " must be the explicit JVM system property false"); + } + } + + private static void deleteTree(Path root) throws IOException { + if (!Files.exists(root)) { + return; + } + try (Stream paths = Files.walk(root)) { + List ordered = paths.sorted(Comparator.reverseOrder()).collect(Collectors.toList()); + for (Path path : ordered) { + Files.deleteIfExists(path); + } + } + } + + private static final class Options { + private final Map values = new LinkedHashMap<>(); + private final List flags = new ArrayList<>(); + + static Options parse(String[] arguments, int start) { + Options result = new Options(); + for (int index = start; index < arguments.length; index++) { + String option = arguments[index]; + if ("--require-owned".equals(option)) { + result.flags.add(option); + continue; + } + if (!option.startsWith("--") || index + 1 >= arguments.length) { + throw new IllegalArgumentException("invalid option: " + option); + } + String value = arguments[++index]; + if (result.values.put(option, value) != null) { + throw new IllegalArgumentException("duplicate option: " + option); + } + } + return result; + } + + Path requiredPath(String name) { + String value = values.remove(name); + if (value == null) { + throw new IllegalArgumentException("missing required option " + name); + } + return Path.of(value); + } + + String optional(String name) { + return values.remove(name); + } + + boolean flag(String name) { + return flags.remove(name); + } + + void requireNoUnused() { + if (!values.isEmpty() || !flags.isEmpty()) { + throw new IllegalArgumentException( + "unknown options: " + values.keySet() + flags); + } + } + } +} diff --git a/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/ParquetEvidence.java b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/ParquetEvidence.java new file mode 100644 index 0000000..cededf8 --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/src/main/java/org/julialang/parquet/n5/ParquetEvidence.java @@ -0,0 +1,754 @@ +package org.julialang.parquet.n5; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericEnumSymbol; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; +import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.avro.AvroSchemaConverter; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ColumnReadStore; +import org.apache.parquet.column.ColumnReader; +import org.apache.parquet.column.impl.ColumnReadStoreImpl; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.conf.PlainParquetConfiguration; +import org.apache.parquet.example.DummyRecordConverter; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.convert.GroupRecordConverter; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.LocalInputFile; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.RecordReader; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; + +final class ParquetEvidence { + private static final String AVRO_READ_SCHEMA_PROPERTY = "parquet.avro.read.schema"; + + private static final class HarnessFailure extends RuntimeException { + HarnessFailure(String message, Throwable cause) { + super(message, cause); + } + } + + static final class ColumnEvidence { + final ColumnDescriptor descriptor; + final List repetition = new ArrayList<>(); + final List definition = new ArrayList<>(); + final List dense = new ArrayList<>(); + final List> pages = new ArrayList<>(); + final List> dictionaries = new ArrayList<>(); + final List rowGroups = new ArrayList<>(); + + ColumnEvidence(ColumnDescriptor descriptor) { + this.descriptor = descriptor; + } + + String path() { + return String.join(".", descriptor.getPath()); + } + + Map toMap() { + PrimitiveType primitive = descriptor.getPrimitiveType(); + return Json.object( + "path", path(), + "physical_type", primitive.getPrimitiveTypeName().name(), + "logical_type", primitive.getLogicalTypeAnnotation() == null + ? null : primitive.getLogicalTypeAnnotation().toString(), + "type_length", primitive.getTypeLength(), + "max_repetition_level", descriptor.getMaxRepetitionLevel(), + "max_definition_level", descriptor.getMaxDefinitionLevel(), + "repetition", repetition, + "definition", definition, + "dense", dense, + "dictionaries", dictionaries, + "pages", pages, + "row_groups", rowGroups.stream() + .map(ColumnRowGroupEvidence::toMap) + .collect(java.util.stream.Collectors.toList())); + } + } + + static final class ColumnRowGroupEvidence { + final int rowGroup; + final long rows; + final List repetition = new ArrayList<>(); + final List definition = new ArrayList<>(); + final List dense = new ArrayList<>(); + final List> pages = new ArrayList<>(); + final List> dictionaries = new ArrayList<>(); + + ColumnRowGroupEvidence(int rowGroup, long rows) { + this.rowGroup = rowGroup; + this.rows = rows; + } + + Map toMap() { + return Json.object( + "row_group", rowGroup, + "rows", rows, + "repetition", repetition, + "definition", definition, + "dense", dense, + "dictionaries", dictionaries, + "pages", pages); + } + } + + static final class AvroEvidence { + final String status; + final String readSchema; + final String materializedSchema; + final List rows; + final List normalizedRows; + final String errorClass; + final String errorMessage; + final List errorStack; + final List> exceptionChain; + + AvroEvidence( + String status, + String readSchema, + String materializedSchema, + List rows, + List normalizedRows, + String errorClass, + String errorMessage, + List errorStack, + List> exceptionChain) { + this.status = status; + this.readSchema = readSchema; + this.materializedSchema = materializedSchema; + this.rows = rows; + this.normalizedRows = normalizedRows; + this.errorClass = errorClass; + this.errorMessage = errorMessage; + this.errorStack = errorStack; + this.exceptionChain = exceptionChain; + } + + Map toMap() { + return Json.object( + "status", status, + "read_schema", readSchema, + "materialized_schema", materializedSchema, + "rows", rows, + "normalized_rows", normalizedRows, + "error_class", errorClass, + "error_message", errorMessage, + "error_stack", errorStack, + "exception_chain", exceptionChain); + } + } + + static final class FileEvidence { + final Path source; + final String file; + final String sha256; + final MessageType schema; + final String createdBy; + final Map metadata; + final long rowCount; + final List> rowGroups; + final List rawRows; + final List columns; + final AvroEvidence avro; + final AvroEvidence explicitAvro; + + FileEvidence( + Path source, + String file, + String sha256, + MessageType schema, + String createdBy, + Map metadata, + long rowCount, + List> rowGroups, + List rawRows, + List columns, + AvroEvidence avro, + AvroEvidence explicitAvro) { + this.source = source; + this.file = file; + this.sha256 = sha256; + this.schema = schema; + this.createdBy = createdBy; + this.metadata = metadata; + this.rowCount = rowCount; + this.rowGroups = rowGroups; + this.rawRows = rawRows; + this.columns = columns; + this.avro = avro; + this.explicitAvro = explicitAvro; + } + + String caseId() { + return metadata.get(OracleMain.CASE_METADATA_KEY); + } + + String pageVersion() { + return metadata.get(OracleMain.PAGE_VERSION_METADATA_KEY); + } + + ColumnEvidence column(String path) { + for (ColumnEvidence column : columns) { + if (column.path().equals(path)) { + return column; + } + } + return null; + } + + Map toMap() { + List> flattened = new ArrayList<>(); + for (ColumnEvidence column : columns) { + flattened.add(column.toMap()); + } + return Json.object( + "record", "file", + "file", file, + "sha256", sha256, + "case_id", caseId(), + "page_version", pageVersion(), + "created_by", createdBy, + "row_count", rowCount, + "metadata", metadata, + "physical_schema", schema.toString(), + "row_groups", rowGroups, + "raw_group_rows", rawRows, + "columns", flattened, + "avro", Json.object( + "add_list_element_records", false, + "inferred", avro.toMap(), + "explicit", explicitAvro == null ? null : explicitAvro.toMap())); + } + } + + private ParquetEvidence() {} + + static FileEvidence inspect( + Path source, + String file, + FixtureCases.AvroNormalizer normalizer, + String explicitAvroSchema) throws IOException { + Path absolute = source.toAbsolutePath().normalize(); + ParquetMetadata footer; + try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(absolute))) { + footer = reader.getFooter(); + } + MessageType schema = footer.getFileMetaData().getSchema(); + String createdBy = footer.getFileMetaData().getCreatedBy(); + Map metadata = new TreeMap<>(footer.getFileMetaData().getKeyValueMetaData()); + List> rowGroups = rowGroupEvidence(footer.getBlocks()); + long rowCount = 0; + for (BlockMetaData block : footer.getBlocks()) { + rowCount = Math.addExact(rowCount, block.getRowCount()); + } + List rawRows = readRawRows(absolute, schema); + if (rowCount != rawRows.size()) { + throw new IllegalStateException( + "raw Group row count " + rawRows.size() + " differs from footer " + rowCount); + } + List columns = readColumns(absolute, footer, schema, createdBy); + readPages(absolute, schema, columns); + AvroEvidence avro = readAvro(absolute, normalizer, null); + AvroEvidence explicitAvro = explicitAvroSchema == null + ? null : readAvro(absolute, row -> row, explicitAvroSchema); + return new FileEvidence( + absolute, + file, + sha256(absolute), + schema, + createdBy, + metadata, + rowCount, + rowGroups, + rawRows, + columns, + avro, + explicitAvro); + } + + private static List> rowGroupEvidence(List blocks) { + List> result = new ArrayList<>(); + for (int blockIndex = 0; blockIndex < blocks.size(); blockIndex++) { + BlockMetaData block = blocks.get(blockIndex); + List> chunks = new ArrayList<>(); + for (ColumnChunkMetaData chunk : block.getColumns()) { + chunks.add(Json.object( + "path", chunk.getPath().toDotString(), + "value_count", chunk.getValueCount(), + "codec", chunk.getCodec().name(), + "encodings", sortedStrings(chunk.getEncodings()), + "total_compressed_size", chunk.getTotalSize(), + "total_uncompressed_size", chunk.getTotalUncompressedSize())); + } + result.add(Json.object( + "ordinal", blockIndex, + "row_count", block.getRowCount(), + "total_byte_size", block.getTotalByteSize(), + "columns", chunks)); + } + return result; + } + + private static List sortedStrings(Collection values) { + List result = new ArrayList<>(); + for (Object value : values) { + result.add(value.toString()); + } + CollectionsSupport.sort(result); + return result; + } + + private static List readRawRows(Path source, MessageType schema) throws IOException { + List result = new ArrayList<>(); + MessageColumnIO columnIo = new ColumnIOFactory().getColumnIO(schema); + try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(source))) { + PageReadStore pages; + while ((pages = reader.readNextRowGroup()) != null) { + RecordReader records = + columnIo.getRecordReader(pages, new GroupRecordConverter(schema)); + for (long index = 0; index < pages.getRowCount(); index++) { + result.add(canonicalGroup(records.read(), schema)); + } + } + } + return result; + } + + private static List readColumns( + Path source, ParquetMetadata footer, MessageType schema, String createdBy) throws IOException { + List result = new ArrayList<>(); + for (ColumnDescriptor descriptor : schema.getColumns()) { + result.add(new ColumnEvidence(descriptor)); + } + try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(source))) { + PageReadStore pages; + int rowGroup = 0; + while ((pages = reader.readNextRowGroup()) != null) { + ColumnReadStore store = new ColumnReadStoreImpl( + pages, new DummyRecordConverter(schema).getRootConverter(), schema, createdBy); + BlockMetaData block = footer.getBlocks().get(rowGroup); + if (block.getColumns().size() != result.size()) { + throw new IllegalStateException("row-group physical column count changed"); + } + for (int columnIndex = 0; columnIndex < result.size(); columnIndex++) { + ColumnEvidence evidence = result.get(columnIndex); + ColumnRowGroupEvidence group = + new ColumnRowGroupEvidence(rowGroup, block.getRowCount()); + ColumnReader column = store.getColumnReader(evidence.descriptor); + long valueCount = block.getColumns().get(columnIndex).getValueCount(); + for (long valueIndex = 0; valueIndex < valueCount; valueIndex++) { + int repetition = column.getCurrentRepetitionLevel(); + int definition = column.getCurrentDefinitionLevel(); + evidence.repetition.add(repetition); + evidence.definition.add(definition); + group.repetition.add(repetition); + group.definition.add(definition); + if (definition == evidence.descriptor.getMaxDefinitionLevel()) { + String dense = canonicalColumnValue(column, evidence.descriptor.getPrimitiveType()); + evidence.dense.add(dense); + group.dense.add(dense); + } + column.consume(); + } + evidence.rowGroups.add(group); + } + rowGroup++; + } + if (rowGroup != footer.getBlocks().size()) { + throw new IllegalStateException("row-group count changed while reading columns"); + } + } + return result; + } + + private static void readPages( + Path source, MessageType schema, List columns) throws IOException { + try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(source))) { + PageReadStore pages; + int rowGroup = 0; + while ((pages = reader.readNextRowGroup()) != null) { + for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { + ColumnEvidence evidence = columns.get(columnIndex); + if (rowGroup >= evidence.rowGroups.size()) { + throw new IllegalStateException("page reader has more row groups than column evidence"); + } + ColumnRowGroupEvidence group = evidence.rowGroups.get(rowGroup); + PageReader pageReader = pages.getPageReader(evidence.descriptor); + DictionaryPage dictionary = pageReader.readDictionaryPage(); + if (dictionary != null) { + Map dictionaryEvidence = Json.object( + "row_group", rowGroup, + "value_count", dictionary.getDictionarySize(), + "encoding", dictionary.getEncoding().name(), + "compressed_size", dictionary.getCompressedSize(), + "uncompressed_size", dictionary.getUncompressedSize()); + evidence.dictionaries.add(dictionaryEvidence); + group.dictionaries.add(dictionaryEvidence); + } + int ordinal = 0; + int valueOffset = 0; + DataPage page; + while ((page = pageReader.readPage()) != null) { + int valueEnd = Math.addExact(valueOffset, page.getValueCount()); + if (valueEnd > group.repetition.size()) { + throw new IllegalStateException("page value count exceeds column stream"); + } + int derivedRowCount = 0; + for (int valueIndex = valueOffset; valueIndex < valueEnd; valueIndex++) { + if (group.repetition.get(valueIndex) == 0) { + derivedRowCount++; + } + } + Map pageEvidence = + pageEvidence(page, rowGroup, ordinal, derivedRowCount); + evidence.pages.add(pageEvidence); + group.pages.add(pageEvidence); + valueOffset = valueEnd; + ordinal++; + } + if (valueOffset != group.repetition.size()) { + throw new IllegalStateException("page value count differs from row-group column stream"); + } + } + rowGroup++; + } + } + for (ColumnEvidence column : columns) { + int repetitionCount = 0; + for (ColumnRowGroupEvidence group : column.rowGroups) { + repetitionCount = Math.addExact(repetitionCount, group.repetition.size()); + } + if (repetitionCount != column.repetition.size()) { + throw new IllegalStateException("row-group streams do not span the column stream"); + } + } + } + + private static Map pageEvidence( + DataPage page, int rowGroup, int ordinal, int derivedRowCount) { + if (page instanceof DataPageV1) { + DataPageV1 v1 = (DataPageV1) page; + Integer indexRowCount = v1.getIndexRowCount().orElse(null); + if (indexRowCount != null && indexRowCount != derivedRowCount) { + throw new IllegalStateException("V1 page row count differs from repetition stream"); + } + return Json.object( + "row_group", rowGroup, + "ordinal", ordinal, + "type", "DATA_PAGE_V1", + "value_count", v1.getValueCount(), + "row_count", derivedRowCount, + "index_row_count", indexRowCount, + "null_count", null, + "encoding", v1.getValueEncoding().name(), + "compressed_size", v1.getCompressedSize(), + "uncompressed_size", v1.getUncompressedSize()); + } + DataPageV2 v2 = (DataPageV2) page; + if (v2.getRowCount() != derivedRowCount) { + throw new IllegalStateException("V2 page row count differs from repetition stream"); + } + return Json.object( + "row_group", rowGroup, + "ordinal", ordinal, + "type", "DATA_PAGE_V2", + "value_count", v2.getValueCount(), + "row_count", v2.getRowCount(), + "index_row_count", null, + "null_count", v2.getNullCount(), + "encoding", v2.getDataEncoding().name(), + "compressed_size", v2.getCompressedSize(), + "uncompressed_size", v2.getUncompressedSize()); + } + + private static AvroEvidence readAvro( + Path source, FixtureCases.AvroNormalizer normalizer, String readSchema) { + PlainParquetConfiguration configuration = new PlainParquetConfiguration(); + configuration.setBoolean(AvroSchemaConverter.ADD_LIST_ELEMENT_RECORDS, false); + if (readSchema != null) { + configuration.set(AVRO_READ_SCHEMA_PROPERTY, readSchema); + } + List rows = new ArrayList<>(); + List normalized = new ArrayList<>(); + String materializedSchema = null; + try (ParquetReader reader = + AvroParquetReader.genericRecordReader(new LocalInputFile(source), configuration)) { + GenericRecord row; + while ((row = reader.read()) != null) { + if (materializedSchema == null) { + materializedSchema = row.getSchema().toString(false); + } + try { + rows.add(Json.encode(canonicalAvro(row))); + normalized.add(Json.encode(canonicalAvro(normalizer.normalize(row)))); + } catch (RuntimeException error) { + throw new HarnessFailure("Avro evidence normalization failed", error); + } + } + return new AvroEvidence( + "success", + readSchema, + materializedSchema, + rows, + normalized, + null, + null, + List.of(), + List.of()); + } catch (HarnessFailure error) { + throw error; + } catch (IOException | RuntimeException error) { + Throwable cause = rootCause(error); + return new AvroEvidence( + "rejected", + readSchema, + materializedSchema, + rows, + normalized, + cause.getClass().getName(), + sanitize(cause.getMessage(), source), + stackEvidence(cause, source), + exceptionChain(error, source)); + } + } + + private static List stackEvidence(Throwable error, Path source) { + List result = new ArrayList<>(); + for (StackTraceElement frame : error.getStackTrace()) { + result.add(sanitize(frame.toString(), source)); + } + return result; + } + + private static List> exceptionChain(Throwable error, Path source) { + List> result = new ArrayList<>(); + Throwable current = error; + while (current != null) { + result.add(Json.object( + "class", current.getClass().getName(), + "message", sanitize(current.getMessage(), source))); + Throwable next = current.getCause(); + current = next == current ? null : next; + } + return result; + } + + private static String sanitize(String value, Path source) { + if (value == null) { + return null; + } + return value.replace(source.toAbsolutePath().toString(), "") + .replace('\n', ' ').replace('\r', ' '); + } + + static String canonicalGroup(Group group, GroupType type) { + StringBuilder result = new StringBuilder("G{"); + for (int fieldIndex = 0; fieldIndex < type.getFieldCount(); fieldIndex++) { + if (fieldIndex > 0) { + result.append(','); + } + Type field = type.getType(fieldIndex); + result.append(field.getName()).append('='); + int count = group.getFieldRepetitionCount(fieldIndex); + if (field.isRepetition(Type.Repetition.REPEATED)) { + result.append('['); + for (int valueIndex = 0; valueIndex < count; valueIndex++) { + if (valueIndex > 0) { + result.append(','); + } + appendGroupValue(result, group, field, fieldIndex, valueIndex); + } + result.append(']'); + } else if (count == 0) { + result.append("null"); + } else { + appendGroupValue(result, group, field, fieldIndex, 0); + } + } + return result.append('}').toString(); + } + + static Object canonicalAvro(Object value) { + if (value == null || value instanceof Boolean || value instanceof Number || value instanceof String) { + return value; + } + if (value instanceof Utf8 || value instanceof GenericEnumSymbol) { + return value.toString(); + } + if (value instanceof GenericRecord) { + GenericRecord record = (GenericRecord) value; + Map result = new LinkedHashMap<>(); + for (org.apache.avro.Schema.Field field : record.getSchema().getFields()) { + result.put(field.name(), canonicalAvro(record.get(field.name()))); + } + return result; + } + if (value instanceof Map) { + Map result = new TreeMap<>(); + for (Map.Entry entry : ((Map) value).entrySet()) { + result.put(entry.getKey().toString(), canonicalAvro(entry.getValue())); + } + return result; + } + if (value instanceof Iterable) { + List result = new ArrayList<>(); + for (Object element : (Iterable) value) { + result.add(canonicalAvro(element)); + } + return result; + } + if (value instanceof ByteBuffer) { + ByteBuffer bytes = ((ByteBuffer) value).duplicate(); + byte[] output = new byte[bytes.remaining()]; + bytes.get(output); + return "base64:" + Base64.getEncoder().encodeToString(output); + } + if (value instanceof GenericFixed) { + return "base64:" + Base64.getEncoder().encodeToString(((GenericFixed) value).bytes()); + } + return value.toString(); + } + + private static void appendGroupValue( + StringBuilder result, Group group, Type field, int fieldIndex, int valueIndex) { + if (!field.isPrimitive()) { + result.append(canonicalGroup(group.getGroup(fieldIndex, valueIndex), field.asGroupType())); + return; + } + PrimitiveType primitive = field.asPrimitiveType(); + switch (primitive.getPrimitiveTypeName()) { + case BOOLEAN: + result.append("bool:").append(group.getBoolean(fieldIndex, valueIndex)); + return; + case INT32: + result.append("i32:").append(group.getInteger(fieldIndex, valueIndex)); + return; + case INT64: + result.append("i64:").append(group.getLong(fieldIndex, valueIndex)); + return; + case FLOAT: + result.append("f32:").append(Float.toHexString(group.getFloat(fieldIndex, valueIndex))); + return; + case DOUBLE: + result.append("f64:").append(Double.toHexString(group.getDouble(fieldIndex, valueIndex))); + return; + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + case INT96: + Binary binary = group.getBinary(fieldIndex, valueIndex); + if (primitive.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { + result.append("utf8:").append(binary.toStringUsingUTF8()); + } else { + result.append("hex:").append(hex(binary.getBytes())); + } + return; + default: + throw new IllegalStateException("unsupported primitive: " + primitive); + } + } + + private static String canonicalColumnValue(ColumnReader column, PrimitiveType primitive) { + switch (primitive.getPrimitiveTypeName()) { + case BOOLEAN: + return Boolean.toString(column.getBoolean()); + case INT32: + return Integer.toString(column.getInteger()); + case INT64: + return Long.toString(column.getLong()); + case FLOAT: + return Float.toHexString(column.getFloat()); + case DOUBLE: + return Double.toHexString(column.getDouble()); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + case INT96: + Binary binary = column.getBinary(); + if (primitive.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { + return "utf8:" + binary.toStringUsingUTF8(); + } + return "hex:" + hex(binary.getBytes()); + default: + throw new IllegalStateException("unsupported primitive: " + primitive); + } + } + + private static String sha256(Path source) throws IOException { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException error) { + throw new IllegalStateException("SHA-256 is unavailable", error); + } + byte[] buffer = new byte[64 * 1024]; + try (InputStream input = Files.newInputStream(source)) { + int count; + while ((count = input.read(buffer)) >= 0) { + if (count > 0) { + digest.update(buffer, 0, count); + } + } + } + return hex(digest.digest()); + } + + private static String hex(byte[] bytes) { + char[] digits = "0123456789abcdef".toCharArray(); + char[] result = new char[bytes.length * 2]; + for (int index = 0; index < bytes.length; index++) { + int value = bytes[index] & 0xff; + result[index * 2] = digits[value >>> 4]; + result[index * 2 + 1] = digits[value & 0x0f]; + } + return new String(result); + } + + private static Throwable rootCause(Throwable error) { + Throwable current = error; + while (current.getCause() != null && current.getCause() != current) { + current = current.getCause(); + } + return current; + } + + private static final class CollectionsSupport { + private CollectionsSupport() {} + + static void sort(List values) { + values.sort(String::compareTo); + } + } +} diff --git a/test/conformance/n5/oracles/parquet-java/src/test/java/org/julialang/parquet/n5/OracleHarnessTest.java b/test/conformance/n5/oracles/parquet-java/src/test/java/org/julialang/parquet/n5/OracleHarnessTest.java new file mode 100644 index 0000000..809b421 --- /dev/null +++ b/test/conformance/n5/oracles/parquet-java/src/test/java/org/julialang/parquet/n5/OracleHarnessTest.java @@ -0,0 +1,165 @@ +package org.julialang.parquet.n5; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.format.Encoding; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.LocalOutputFile; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +final class OracleHarnessTest { + @TempDir Path temporary; + + @Test + void canonicalEncodingOrderUsesNumericValuesAndPreservesDuplicates() { + assertEquals( + List.of(Encoding.PLAIN, Encoding.RLE, Encoding.RLE), + CanonicalParquetFooter.sortedEncodings( + List.of(Encoding.RLE, Encoding.PLAIN, Encoding.RLE))); + } + + @Test + void generatesDeterministicSelfVerifiedV1AndV2Fixtures() throws Exception { + List evidence = OracleMain.selfTest(temporary); + assertEquals(FixtureCases.all().size() * 2, evidence.size()); + for (ParquetEvidence.FileEvidence file : evidence) { + assertTrue(Files.isRegularFile(temporary.resolve("first").resolve(file.file))); + assertEquals(64, file.sha256.length()); + assertEquals("false", System.getProperty("parquet.avro.add-list-element-records")); + } + for (String version : List.of("v1", "v2")) { + ParquetEvidence.FileEvidence rule3 = evidence.stream() + .filter(file -> file.file.equals("list_rule3_nested." + version + ".parquet")) + .findFirst() + .orElseThrow(); + ParquetEvidence.ColumnEvidence column = rule3.column("items.array.array"); + assertEquals(List.of(0, 0, 0, 0, 2, 1, 1), column.repetition); + assertEquals(List.of(0, 1, 2, 3, 3, 2, 3), column.definition); + assertEquals(List.of("1", "2", "3"), column.dense); + assertEquals(1, column.rowGroups.size()); + assertEquals(column.repetition, column.rowGroups.get(0).repetition); + assertEquals(column.definition, column.rowGroups.get(0).definition); + assertEquals(column.dense, column.rowGroups.get(0).dense); + assertEquals("success", rule3.avro.status); + assertEquals( + List.of( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[[]]}", + "{\"items\":[[1,2],[],[3]]}"), + rule3.avro.normalizedRows); + assertNull(rule3.explicitAvro); + + ParquetEvidence.FileEvidence diagnostic = evidence.stream() + .filter(file -> file.file.equals( + "list_rule3_unannotated_diagnostic." + version + ".parquet")) + .findFirst() + .orElseThrow(); + assertEquals("rejected", diagnostic.avro.status); + assertEquals("java.lang.ClassCastException", diagnostic.avro.errorClass); + assertEquals("repeated int32 element is not a group", diagnostic.avro.errorMessage); + assertTrue(!diagnostic.avro.errorStack.isEmpty()); + assertNull(diagnostic.explicitAvro); + + ParquetEvidence.FileEvidence listMap = evidence.stream() + .filter(file -> file.file.equals("list_direct_map_utf8." + version + ".parquet")) + .findFirst() + .orElseThrow(); + assertEquals(List.of("utf8:a", "utf8:a", "utf8:b"), + listMap.column("items.map.key_value.key").dense); + assertEquals("success", listMap.avro.status); + assertEquals( + "{\"type\":\"record\",\"name\":\"list_direct_map_utf8\",\"fields\":[" + + "{\"name\":\"items\",\"type\":[\"null\",{\"type\":\"array\"," + + "\"items\":{\"type\":\"map\",\"values\":\"int\"}}]," + + "\"default\":null}]}", + listMap.avro.materializedSchema); + assertEquals( + List.of( + "{\"items\":null}", + "{\"items\":[]}", + "{\"items\":[{}]}", + "{\"items\":[{\"a\":20},{},{\"b\":30}]}"), + listMap.avro.normalizedRows); + } + } + + @Test + void strictVerificationBindsUnownedFilesAndPreservesRowGroups() throws Exception { + FixtureCases.CaseSpec spec = FixtureCases.find("list_rule1_primitive"); + Path source = temporary.resolve("unbound.parquet"); + writeUnbound(source, spec); + + ParquetEvidence.FileEvidence inspected = + ParquetEvidence.inspect(source, source.getFileName().toString(), spec.avroNormalizer, null); + ParquetEvidence.ColumnEvidence column = inspected.column("items.element"); + assertEquals(2, column.rowGroups.size()); + assertEquals(List.of(0, 0), column.rowGroups.get(0).repetition); + assertEquals(List.of(0, 1), column.rowGroups.get(0).definition); + assertEquals(List.of(), column.rowGroups.get(0).dense); + assertEquals(List.of(0, 0, 1), column.rowGroups.get(1).repetition); + assertEquals(List.of(2, 2, 2), column.rowGroups.get(1).definition); + assertEquals(List.of("10", "20", "30"), column.rowGroups.get(1).dense); + + assertThrows(IllegalStateException.class, + () -> OracleMain.verifyFiles(source, null, null)); + List verified = + OracleMain.verifyFiles(source, spec.id, "v1"); + assertEquals(1, verified.size()); + assertEquals(2, verified.get(0).column("items.element").rowGroups.size()); + assertThrows(IllegalStateException.class, + () -> OracleMain.verifyFiles(source, spec.id, "v2")); + assertThrows(IllegalArgumentException.class, + () -> OracleMain.verifyFiles(source, "missing-case", "v1")); + } + + @Test + void auditRecordsEverySupportedAndUnsupportedFile() throws Exception { + FixtureCases.CaseSpec spec = FixtureCases.find("list_rule1_primitive"); + Path valid = temporary.resolve("valid.parquet"); + Path invalid = temporary.resolve("invalid.parquet"); + writeUnbound(valid, spec); + Files.write(invalid, new byte[] {0, 1, 2, 3}); + + List> records = OracleMain.auditRecords(temporary); + assertEquals(3, records.size()); + assertEquals("run", records.get(0).get("record")); + assertEquals(2, records.get(0).get("file_count")); + assertEquals(1, records.get(0).get("supported_count")); + assertEquals(1, records.get(0).get("unsupported_count")); + assertEquals("unsupported", records.get(1).get("record")); + assertEquals("invalid.parquet", records.get(1).get("file")); + assertEquals("file", records.get(2).get("record")); + assertEquals("valid.parquet", records.get(2).get("file")); + } + + private static void writeUnbound(Path output, FixtureCases.CaseSpec spec) throws Exception { + try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(output)) + .withType(spec.schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .withRowGroupSize(1024 * 1024) + .withRowGroupRowCountLimit(2) + .withPageSize(1024 * 1024) + .withDictionaryEncoding(false) + .withValidation(true) + .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0) + .withPageWriteChecksumEnabled(true) + .build()) { + for (Group row : spec.createRows()) { + writer.write(row); + } + } + } +} diff --git a/test/conformance/n5/run-oracle-container.sh b/test/conformance/n5/run-oracle-container.sh new file mode 100755 index 0000000..05d535a --- /dev/null +++ b/test/conformance/n5/run-oracle-container.sh @@ -0,0 +1,195 @@ +#!/bin/sh +set -eu + +usage() { + echo "usage: run-oracle-container.sh --image IMAGE --repo REPO --corpus CORPUS --output DIR" >&2 + exit 64 +} + +fail() { + echo "run-oracle-container.sh: $1" >&2 + exit 65 +} + +host_sha256() { + n5_hash_file=$1 + [ -f "$n5_hash_file" ] || fail "required file $n5_hash_file is absent" + if command -v sha256sum >/dev/null 2>&1; then + n5_hash_output=$(sha256sum "$n5_hash_file") || + fail "cannot hash $n5_hash_file" + else + n5_hash_output=$(shasum -a 256 "$n5_hash_file") || + fail "cannot hash $n5_hash_file" + fi + n5_hash_value=${n5_hash_output%% *} + case "$n5_hash_value" in + *[!0-9a-f]*) + fail "invalid SHA-256 output for $n5_hash_file" + ;; + esac + [ "${#n5_hash_value}" -eq 64 ] || + fail "invalid SHA-256 output for $n5_hash_file" + printf '%s\n' "$n5_hash_value" +} + +verify_evidence() { + n5_evidence_root=$1 + if command -v sha256sum >/dev/null 2>&1; then + (cd "$n5_evidence_root" && + sha256sum --check --quiet --strict evidence.sha256) + else + (cd "$n5_evidence_root" && shasum -a 256 --check evidence.sha256) + fi +} + +n5_image= +n5_repo= +n5_corpus= +n5_output= +while [ "$#" -gt 0 ]; do + case "$1" in + --image) + [ "$#" -ge 2 ] || usage + n5_image=$2 + shift 2 + ;; + --repo) + [ "$#" -ge 2 ] || usage + n5_repo=$2 + shift 2 + ;; + --corpus) + [ "$#" -ge 2 ] || usage + n5_corpus=$2 + shift 2 + ;; + --output) + [ "$#" -ge 2 ] || usage + n5_output=$2 + shift 2 + ;; + *) usage ;; + esac +done +[ -n "$n5_image" ] && [ -n "$n5_repo" ] && [ -n "$n5_corpus" ] && + [ -n "$n5_output" ] || usage +command -v docker >/dev/null 2>&1 || fail "Docker is required" +[ -d "$n5_repo/test/conformance/n5" ] || fail "repository root is absent" +[ -d "$n5_corpus/.git" ] || fail "parquet-testing checkout is absent" +n5_repo=$(CDPATH= cd -- "$n5_repo" && pwd -P) || fail "cannot resolve repository root" +n5_corpus=$(CDPATH= cd -- "$n5_corpus" && pwd -P) || fail "cannot resolve corpus root" +n5_output_parent=$(dirname -- "$n5_output") +n5_output_name=$(basename -- "$n5_output") +case "$n5_output_name" in + '' | . | .. | *[!A-Za-z0-9._-]*) + fail "output must have a portable child-directory name" + ;; +esac +mkdir -p "$n5_output_parent" +n5_output_parent=$(CDPATH= cd -- "$n5_output_parent" && pwd -P) || + fail "cannot resolve output parent" +case "$n5_repo$n5_corpus$n5_output_parent" in + *','*) fail "bind-mount paths must not contain a comma" ;; +esac +n5_output="$n5_output_parent/$n5_output_name" +if [ -e "$n5_output" ]; then + [ ! -L "$n5_output" ] || fail "output must not be a symbolic link" + [ -d "$n5_output" ] || fail "output is not a directory" + n5_output_entry=$(find "$n5_output" -mindepth 1 -print -quit) || + fail "cannot inspect output directory" + [ -z "$n5_output_entry" ] || + fail "output directory must be empty" +fi +n5_host_stage=$(mktemp -d \ + "$n5_output_parent/.n5-$n5_output_name-stage.XXXXXX") || + fail "cannot create host oracle stage" +cleanup() { + rm -rf "$n5_host_stage" +} +preserve_stage() { + echo "run-oracle-container.sh: $1; preserved at $n5_host_stage" >&2 + trap - EXIT HUP INT TERM + exit 73 +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +n5_canary="$n5_repo/test/conformance/n5/manifest.toml" +n5_canary_before=$(host_sha256 "$n5_canary") +if docker run --rm \ + --platform linux/amd64 \ + --network none \ + --mount "type=bind,src=$n5_repo,dst=/work/repo,readonly" \ + --mount "type=bind,src=$n5_corpus,dst=/work/parquet-testing,readonly" \ + --mount "type=bind,src=$n5_host_stage,dst=/work/output-stage" \ + "$n5_image" \ + /bin/sh -c ' + if printf "%s\n" n5-write-must-fail >> "$1" 2>/dev/null; then + echo "run-oracle-container.sh: repository canary was writable" >&2 + exit 70 + fi + export N5_READONLY_CANARY_VERIFIED=1 + shift + exec "$@" + ' n5-readonly-canary \ + /work/repo/test/conformance/n5/manifest.toml \ + /work/repo/test/conformance/n5/oracle-gate.sh \ + --repo /work/repo \ + --corpus /work/parquet-testing \ + --output /work/output-stage/result \ + > "$n5_host_stage/container.log" 2>&1; then + n5_container_status=0 +else + n5_container_status=$? +fi +n5_canary_after=$(host_sha256 "$n5_canary") || + preserve_stage "cannot verify the repository canary after the run" +if [ "$n5_canary_after" != "$n5_canary_before" ]; then + n5_container_status=73 + n5_host_failure="read-only repository canary changed" +else + n5_host_failure= +fi +n5_result="$n5_host_stage/result" +if [ ! -d "$n5_result" ]; then + mkdir "$n5_result" || + preserve_stage "cannot preserve missing gate result" + printf '%s\n' "oracle container did not produce a result" \ + > "$n5_result/failure.txt" || + preserve_stage "cannot record the missing gate result" + [ "$n5_container_status" -ne 0 ] || n5_container_status=73 +fi +if [ "$n5_container_status" -eq 0 ]; then + if [ ! -f "$n5_result/status.json" ] || + [ ! -f "$n5_result/evidence.sha256" ] || + ! verify_evidence "$n5_result"; then + n5_container_status=73 + n5_host_failure="successful oracle result is incomplete or corrupt" + fi +fi +if [ -n "$n5_host_failure" ]; then + rm -f "$n5_result/status.json" "$n5_result/evidence.sha256" || + preserve_stage "cannot remove invalid success markers" + printf '%s\n' "$n5_host_failure" > "$n5_result/host-failure.txt" || + preserve_stage "cannot record the host failure" +fi +if [ "$n5_container_status" -ne 0 ]; then + rm -f "$n5_result/status.json" "$n5_result/evidence.sha256" || + preserve_stage "cannot remove failed success markers" + cp "$n5_host_stage/container.log" "$n5_result/container.log" || + preserve_stage "cannot preserve failed container log" + printf '{"schema_version":1,"container_exit_status":%s}\n' \ + "$n5_container_status" > "$n5_result/container-status.json" || + preserve_stage "cannot record the container exit status" +fi +if [ -d "$n5_output" ]; then + rmdir "$n5_output" || preserve_stage "cannot replace empty output" +fi +if ! mv "$n5_result" "$n5_output"; then + echo "run-oracle-container.sh: cannot commit result; preserved at $n5_result" >&2 + trap - EXIT HUP INT TERM + exit 73 +fi +exit "$n5_container_status" diff --git a/test/conformance/n5/run-oracles.sh b/test/conformance/n5/run-oracles.sh new file mode 100755 index 0000000..65db241 --- /dev/null +++ b/test/conformance/n5/run-oracles.sh @@ -0,0 +1,231 @@ +#!/bin/sh +set -eu + +N5_REPOSITORY=ghcr.io/juliaio/parquet-jl-n5-oracles + +usage() { + echo "usage: run-oracles.sh --lock LOCK --network none" >&2 + exit 64 +} + +fail() { + echo "run-oracles.sh: $1" >&2 + exit 65 +} + +is_hex() { + n5_hex_value=$1 + n5_hex_length=$2 + [ "${#n5_hex_value}" -eq "$n5_hex_length" ] || return 1 + case "$n5_hex_value" in + *[!0-9a-f]*) return 1 ;; + esac + return 0 +} + +is_sha256() { + n5_sha_value=$1 + case "$n5_sha_value" in + sha256:*) n5_sha_hex=${n5_sha_value#sha256:} ;; + *) return 1 ;; + esac + is_hex "$n5_sha_hex" 64 +} + +lock_value() { + n5_lock_key=$1 + n5_lock_result=$(sed -n \ + "s/^${n5_lock_key}[[:space:]]*=[[:space:]]*\"\([^\"]*\)\"[[:space:]]*$/\1/p" \ + "$n5_lock") || fail "cannot read lock field $n5_lock_key" + [ -n "$n5_lock_result" ] || fail "lock field $n5_lock_key is absent" + printf '%s\n' "$n5_lock_result" +} + +expect_lock_value() { + n5_lock_key=$1 + n5_lock_expected=$2 + n5_lock_actual=$(lock_value "$n5_lock_key") + [ "$n5_lock_actual" = "$n5_lock_expected" ] || + fail "lock field $n5_lock_key differs from the exact contract" +} + +host_sha256() { + n5_hash_file=$1 + [ -f "$n5_hash_file" ] || fail "required file $n5_hash_file is absent" + if command -v sha256sum >/dev/null 2>&1; then + n5_hash_output=$(sha256sum "$n5_hash_file") || + fail "cannot hash $n5_hash_file" + else + n5_hash_output=$(shasum -a 256 "$n5_hash_file") || + fail "cannot hash $n5_hash_file" + fi + n5_hash_value=${n5_hash_output%% *} + is_hex "$n5_hash_value" 64 || fail "invalid SHA-256 output for $n5_hash_file" + printf '%s\n' "$n5_hash_value" +} + +image_sha256() { + n5_hash_file=$1 + n5_hash_output=$(docker run --rm --platform linux/amd64 --network none \ + --entrypoint sha256sum "$n5_image" "$n5_hash_file") || + fail "cannot hash $n5_hash_file in $n5_image" + n5_hash_value=${n5_hash_output%% *} + is_hex "$n5_hash_value" 64 || fail "invalid image SHA-256 for $n5_hash_file" + printf '%s\n' "$n5_hash_value" +} + +n5_root=$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd) +n5_lock= +n5_network= +while [ "$#" -gt 0 ]; do + case "$1" in + --lock) + [ "$#" -ge 2 ] || usage + n5_lock=$2 + shift 2 + ;; + --network) + [ "$#" -ge 2 ] || usage + n5_network=$2 + shift 2 + ;; + *) + usage + ;; + esac +done +[ -f "$n5_lock" ] || usage +[ "$n5_network" = "none" ] || { + echo "run-oracles.sh: the binding gate requires --network none" >&2 + exit 64 +} +command -v docker >/dev/null 2>&1 || { + echo "run-oracles.sh: Docker is required" >&2 + exit 69 +} + +n5_work=$(mktemp -d) +cleanup() { + rm -rf "$n5_work" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +printf '%s\n' \ + schema_version \ + platform \ + image_repository \ + image_digest \ + image_reference \ + base_image_digest \ + maven_version \ + maven_archive_sha512 \ + maven_artifacts_manifest_sha256 \ + maven_dependency_tree_sha256 \ + rust_version \ + rust_channel_manifest_sha256 \ + rust_tarball_sha256 \ + cargo_vendor_manifest_sha256 \ + cargo_dependency_tree_sha256 \ + harness_source_manifest_sha256 \ + parquet_java_version \ + parquet_java_commit \ + arrow_rs_version \ + arrow_rs_commit \ + parquet_testing_commit \ + fixture_manifest_sha256 > "$n5_work/expected-keys" +sed -n 's/^\([a-z0-9_][a-z0-9_]*\)[[:space:]]*=.*$/\1/p' "$n5_lock" \ + > "$n5_work/actual-keys" || fail "cannot parse lock keys" +n5_lock_lines=$(wc -l < "$n5_lock") +[ "$n5_lock_lines" -eq 22 ] || fail "lock must contain exactly 22 fields" +cmp -s "$n5_work/expected-keys" "$n5_work/actual-keys" || + fail "lock keys or field order differ from the exact contract" +grep -F -x 'schema_version = 1' "$n5_lock" >/dev/null || + fail "lock schema_version differs from the exact contract" + +expect_lock_value platform linux/amd64 +expect_lock_value image_repository "$N5_REPOSITORY" +expect_lock_value base_image_digest \ + sha256:ab2527b3c9b7c15bc88f60dec19b2aa39939a6e0045fb8f538eeecbd7af59c69 +expect_lock_value maven_version 3.9.8 +expect_lock_value maven_archive_sha512 \ + 7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2 +expect_lock_value rust_version 1.96.1 +expect_lock_value rust_channel_manifest_sha256 \ + 87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975 +expect_lock_value rust_tarball_sha256 \ + d29ccb1559a177c4e72291f6e5f629de7fe8885e7521ca47802627544b121e95 +expect_lock_value parquet_java_version 1.17.1 +expect_lock_value parquet_java_commit 78a8d3230eb4769db93de5f2f2e18363c04cae81 +expect_lock_value arrow_rs_version 59.2.0 +expect_lock_value arrow_rs_commit 782e5a685501a9db6cc8e9a3b7cbff894940c47a +expect_lock_value parquet_testing_commit 09f3cdbde45302f0f0c689c950e465e98a9df960 + +n5_digest=$(lock_value image_digest) +is_sha256 "$n5_digest" || fail "lock has no exact image digest" +n5_image=$(lock_value image_reference) +[ "$n5_image" = "$N5_REPOSITORY@$n5_digest" ] || + fail "image reference does not match the fixed repository and digest" +for n5_hash_key in \ + maven_artifacts_manifest_sha256 \ + maven_dependency_tree_sha256 \ + cargo_vendor_manifest_sha256 \ + cargo_dependency_tree_sha256 \ + harness_source_manifest_sha256 \ + fixture_manifest_sha256; do + n5_hash_value=$(lock_value "$n5_hash_key") + is_hex "$n5_hash_value" 64 || fail "lock field $n5_hash_key is not a SHA-256" +done + +docker image inspect "$n5_image" >/dev/null 2>&1 || + docker pull --platform linux/amd64 "$n5_image" >/dev/null +n5_repo_digests=$(docker image inspect "$n5_image" \ + --format '{{join .RepoDigests "\n"}}') || fail "cannot inspect image RepoDigests" +n5_matched=0 +for n5_candidate in $n5_repo_digests; do + [ "$n5_candidate" = "$n5_image" ] && n5_matched=1 +done +[ "$n5_matched" -eq 1 ] || fail "pulled RepoDigest does not match the lock" +n5_platform=$(docker image inspect "$n5_image" \ + --format '{{.Os}}/{{.Architecture}}') || fail "cannot inspect image platform" +[ "$n5_platform" = "linux/amd64" ] || fail "locked image is not linux/amd64" + +[ "$(image_sha256 /opt/n5/manifests/maven-artifacts.sha256)" = \ + "$(lock_value maven_artifacts_manifest_sha256)" ] || + fail "Maven artifact manifest differs from the lock" +[ "$(image_sha256 /opt/n5/manifests/maven-dependency-tree.txt)" = \ + "$(lock_value maven_dependency_tree_sha256)" ] || + fail "Maven dependency tree differs from the lock" +[ "$(image_sha256 /opt/n5/manifests/cargo-vendor.sha256)" = \ + "$(lock_value cargo_vendor_manifest_sha256)" ] || + fail "Cargo content manifest differs from the lock" +[ "$(image_sha256 /opt/n5/manifests/cargo-dependency-tree.txt)" = \ + "$(lock_value cargo_dependency_tree_sha256)" ] || + fail "Cargo dependency tree differs from the lock" +[ "$(image_sha256 /opt/n5/manifests/harness-source.sha256)" = \ + "$(lock_value harness_source_manifest_sha256)" ] || + fail "harness source manifest differs from the lock" + +n5_expected_manifest=$(lock_value fixture_manifest_sha256) +n5_actual_manifest=$(host_sha256 "$n5_root/test/conformance/n5/manifest.toml") +[ "$n5_actual_manifest" = "$n5_expected_manifest" ] || + fail "fixture manifest differs from the lock" + +n5_corpus=${PARQUET_TESTING_DIR:-"$n5_root/test/parquet-testing"} +[ -d "$n5_corpus/.git" ] || { + echo "run-oracles.sh: pinned parquet-testing checkout is absent" >&2 + exit 66 +} +n5_corpus_commit=$(git -C "$n5_corpus" rev-parse HEAD) || + fail "cannot read parquet-testing commit" +[ "$n5_corpus_commit" = "$(lock_value parquet_testing_commit)" ] || + fail "parquet-testing commit differs from the lock" + +n5_output=${N5_ORACLE_OUTPUT_DIR:-"$n5_root/test/conformance/n5/output"} +"$n5_root/test/conformance/n5/run-oracle-container.sh" \ + --image "$n5_image" \ + --repo "$n5_root" \ + --corpus "$n5_corpus" \ + --output "$n5_output" diff --git a/test/conformance/n5/runtests.jl b/test/conformance/n5/runtests.jl new file mode 100644 index 0000000..efcd7fb --- /dev/null +++ b/test/conformance/n5/runtests.jl @@ -0,0 +1,9 @@ +using Parquet +using Test + +@testset "N5 nested conformance" begin + include("corpus.jl") + include("model/runtests.jl") + include("hardening/source_mutation.jl") + include("external.jl") +end diff --git a/test/conformance/n5/test-oracle-container.sh b/test/conformance/n5/test-oracle-container.sh new file mode 100755 index 0000000..2953b4e --- /dev/null +++ b/test/conformance/n5/test-oracle-container.sh @@ -0,0 +1,89 @@ +#!/bin/sh +set -eu + +usage() { + echo "usage: test-oracle-container.sh --image IMAGE --corpus CORPUS" >&2 + exit 64 +} + +host_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + n5_hash_output=$(sha256sum "$1") || return 1 + else + n5_hash_output=$(shasum -a 256 "$1") || return 1 + fi + printf '%s\n' "${n5_hash_output%% *}" +} + +assert_no_host_stage() { + n5_stage_entry=$(find "$n5_work" -maxdepth 1 \ + -name '.n5-*-stage.*' -print -quit) || return 1 + [ -z "$n5_stage_entry" ] +} + +assert_canary_unchanged() { + n5_canary_after=$(host_sha256 "$n5_canary") || return 1 + [ "$n5_canary_after" = "$n5_canary_before" ] +} + +n5_image= +n5_corpus= +while [ "$#" -gt 0 ]; do + case "$1" in + --image) + [ "$#" -ge 2 ] || usage + n5_image=$2 + shift 2 + ;; + --corpus) + [ "$#" -ge 2 ] || usage + n5_corpus=$2 + shift 2 + ;; + *) usage ;; + esac +done +[ -n "$n5_image" ] && [ -d "$n5_corpus/.git" ] || usage +n5_repo=$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd -P) +n5_work=$(mktemp -d) +cleanup() { + rm -rf "$n5_work" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +n5_canary="$n5_repo/test/conformance/n5/manifest.toml" +n5_canary_before=$(host_sha256 "$n5_canary") +"$n5_repo/test/conformance/n5/run-oracle-container.sh" \ + --image "$n5_image" \ + --repo "$n5_repo" \ + --corpus "$n5_corpus" \ + --output "$n5_work/success" +[ -f "$n5_work/success/status.json" ] +[ -f "$n5_work/success/read-only-canary.json" ] +[ -f "$n5_work/success/evidence.sha256" ] +assert_canary_unchanged +assert_no_host_stage + +cp -R "$n5_corpus" "$n5_work/corrupt-corpus" +printf '%s\n' n5-forced-corruption \ + >> "$n5_work/corrupt-corpus/data/datapage_v2.snappy.parquet" +if "$n5_repo/test/conformance/n5/run-oracle-container.sh" \ + --image "$n5_image" \ + --repo "$n5_repo" \ + --corpus "$n5_work/corrupt-corpus" \ + --output "$n5_work/failure"; then + echo "test-oracle-container.sh: corrupt corpus unexpectedly passed" >&2 + exit 1 +else + n5_status=$? +fi +[ "$n5_status" -eq 65 ] +[ -f "$n5_work/failure/failure.txt" ] +[ -f "$n5_work/failure/container.log" ] +grep -F 'corpus file hash differs' "$n5_work/failure/failure.txt" >/dev/null +assert_canary_unchanged +assert_no_host_stage +echo "oracle container success and rollback checks passed" diff --git a/test/conformance/n6/README.md b/test/conformance/n6/README.md new file mode 100644 index 0000000..80753b2 --- /dev/null +++ b/test/conformance/n6/README.md @@ -0,0 +1,246 @@ +# N6 statistics conformance gate + +This directory owns the test-first evidence for trusted row-group statistics. +It is separate from the frozen N5 corpus and oracle evidence. + +The authority is `docs/dev/n6-statistics-plan.md` at SHA-256 +`15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304`. +Production N6 statistics files must not change until this preproduction gate +has an exact review with no open P0, P1, or P2 finding. + +## Frozen roles + +- `model/` computes order, count, and trust decisions without importing + Parquet.jl or sharing production comparison code. +- `oracles/raw-java/` records Parquet 2.13 Compact-Thrift wire facts in a + separate JVM. Its scanner-specific schema does not claim semantic support. +- `fixtures.toml` gives every selected Apache file a stable case ID, byte + identity, expected topology, capability scope, and evidence status. It also + freezes the exact generated Julia cases and output identities. +- `capabilities.toml` records each authority claim as `verified`, `planned`, + `unsupported`, or `not_assessed`. It binds evidence producers to exact + source revisions and allowed toolchain artifacts. Unsupported never means + pass. +- `evidence.schema.json` is the normalized cross-tool JSONL boundary. It keeps + the complete logical leaf description needed by the independent model. It + also records explicit `PASS`, `FAIL`, and `UNSUPPORTED` case results. +- `validate_evidence.py` meta-validates the Draft 2020-12 schema and enforces + cross-record facts that JSON Schema cannot express. A case result must be + in that producer's reviewed capability scope. +- `manifest.toml` pins source revisions, policy files, runtimes, wheels, + composite producer descriptors, and evidence identities without local + absolute paths. It also pins the deterministic raw scanner digest for the + exact 20-file Apache corpus. That digest is raw evidence only. +- `artifacts.sha256` pins every static preproduction N6 file except itself, + `manifest.toml`, declared evidence outputs, declared generated Parquet files, + and the raw build directory. The manifest pins both the artifact list and + each frozen evidence file. The runner rejects every other unlisted file and + every symbolic link before oracle orchestration. + +The raw scanner output is normalized in a separate reviewed step. A normalizer +may not use Parquet.jl's production decoder or comparator. The raw toolchain, +raw corpus bytes, normalized capability claims, and external Java, Rust, +PyArrow, and DuckDB N6 harnesses are verified. +The raw evidence entry uses `storage = "gate-generated"`: its declared path is +a stable identity label, but the runner writes both comparison scans to fresh +temporary files and does not materialize that path in the repository. + +## One-way evidence binding + +A normalized run contains neither `manifest_sha256` nor +`artifact_manifest_sha256`. Either field could create a checksum cycle when +its containing file is pinned. A run binds these immutable inputs instead: + +- the plan; +- the capability matrix; +- the fixture and corpus manifests; +- the normalized schema; and +- its exact producer toolchain. + +The artifact manifest also excludes every declared evidence path. The +containing manifest separately binds the artifact list and each evidence path, +byte hash, authority, toolchain hash, case count, and record count. The +validator accepts a gate input only when its +canonical repository-relative path selects exactly one `[[frozen_evidence]]` +entry with status `verified` and format `normalized-jsonl`. A +`[[planned_evidence]]` entry can never satisfy `--gate`. + +Every evidence exclusion must be a `.jsonl` file below the exact repository +prefix `test/conformance/n6/evidence/`. Every generated Parquet exclusion must +be a `.parquet` file below the N6-relative prefix `generated/`. The runner +checks both prefix sets and uniqueness before it derives any artifact +exclusion, so a manifest entry cannot hide an existing static N6 file. + +All declared normalized evidence is frozen and verified. The exact gate still +has preproduction status. Publication and oracle locking remain unauthorized +until the final independent review and full exact gate complete. + +## Normalized leaf meaning + +`leaf_schema.logical_type` is the effective leaf annotation. It is not a raw +presence bit. A normalizer uses the raw `LogicalType` member when present. When +it is absent, the normalizer synthesizes the matching effective value from a +legacy `ConvertedType`; for example, `UTF8` becomes `STRING` and `UINT_16` +becomes `INTEGER`. `converted_type` separately preserves the raw legacy value, +or `null` when it was absent. Legacy integer names supply width and signedness. +Legacy decimal parameters come from `SchemaElement`. A legacy-only millisecond +or microsecond time annotation synthesizes `is_adjusted_to_utc = true`. When a +modern temporal annotation is present, its UTC flag wins. Both UTC and local +modern annotations carry the matching legacy enum for forward compatibility; +that enum cannot encode the UTC flag. + +The pinned IDL requires compatible modern annotations to carry their exact +legacy counterpart when one exists. The semantic validator enforces that +pairing, its physical type, and every parameter that the legacy enum can +represent. This includes decimal precision and scale, integer width and +signedness, and temporal unit, but not the modern temporal UTC flag. It also +validates fixed widths and geospatial parameters. `TIME` and `TIMESTAMP` with +`NANOS` have no legacy counterpart. `UUID`, `FLOAT16`, `UNKNOWN`, `VARIANT`, +`GEOMETRY`, and `GEOGRAPHY` also have none. `INTERVAL` is synthesized from its +legacy annotation. `UNKNOWN` means the known Parquet null logical type. An +unrecognized future logical union member cannot be represented as `UNKNOWN`; +normalization must fail until the schema can preserve that member. + +MAP, LIST, and VARIANT annotate groups and are rejected on normalized leaves. +A positive `type_length` is required for `FIXED_LEN_BYTE_ARRAY`. Any signed-i32 +`type_length` on another physical type is preserved without assigning it N6 +statistics meaning. The pinned IDL defines that raw field as an optional +maximum bit length but gives it no validity range. + +## Result digest contracts + +`n6-capability-result-sha256-v1` hashes a UTF-8 JSON observation envelope with +no final newline. Its object is exactly +`{"capability_id":string,"case_id":string,"observations":array}` and uses the +RFC 8785 JSON Canonicalization Scheme. Observations may contain integers but no +floating JSON numbers. The exact producer harness defines and freezes the +ordered contents of `observations` in the verified evidence. + +`n6-no-pruning-trace-sha256-v1` is narrower. The reader harness parses metadata, +sets `footer_start = file_size - footer_length - 8`, clears its source trace, +and then materializes the table. Each later read is clipped to byte interval +`[4, footer_start)`; empty intersections are omitted. In call order, the trace +contains one UTF-8 line `offset=;length=\n` per clipped read. +`range_trace_sha256` hashes those exact lines. `body_sha256` hashes file bytes +`[0, footer_start)`. `logical_values_sha256` hashes the RFC 8785 logical value +array, again with no floating JSON numbers. The final result digest +hashes these exact UTF-8 lines, including the final line feed: + +```text +body_sha256= +logical_values_sha256= +range_trace_sha256= +read_count= +``` + +The five no-pruning variants share one seed, comparison group, and digest +contract. They differ only by the declared statistics mutation. A verified +comparison group must have one equal passing digest across all five variants. +Every generated output has `output_identity_status = "verified"` and an exact +frozen byte hash and size. + +## Bounded validation + +The frozen manifest sets exact limits for input count, bytes per file, total +bytes, bytes per JSONL line, records per input, and total records. The record +limits are derived from all declared fixture topology and capability mappings. +The validator rejects duplicate JSON keys, non-standard numeric constants, +missing final line feeds, oversized inputs, duplicate producers, conflicting +file or leaf facts, column orders that change across row groups, and conflicting +passing digests. + +PyArrow and DuckDB authority digests name composite descriptors under +`toolchains/`. Each descriptor binds the CPython source, exact standalone +distribution archive, clean extraction policy, executable, runtime tree, exact +wheel, platform, and harness sources. Only verified descriptors authorize +passing gate evidence. + +The PyArrow and DuckDB source entries remain `planned`. The gate verifies their +official wheel bytes and runtime behavior, but it does not claim a +cryptographic tagged-source-tree-to-wheel build mapping. This provenance limit +does not weaken the exact wheel, harness, or observed interoperability results. + +The gate requires a PASS for each declared case and capability that has at +least one positive reviewed claim. A pair with only `unsupported` or +`not_assessed` claims stays explicit but is not treated as a pass. Unsupported +never satisfies a positive claim. + +## Local checks + +Run the static gate and independent model from the repository root: + +```sh +julia +1.10.11 --project=. --startup-file=no --history-file=no test/conformance/n6/runtests.jl +julia +1.12.6 --project=. --startup-file=no --history-file=no test/conformance/n6/runtests.jl +``` + +Default CI runs this static preflight with exact Julia 1.10.11 and 1.12.6 on +macOS 15 arm64. It sets `PARQUET_N6_GATE=0`. It does not run the expensive +external gate or any oracle. + +The exact source gate uses environment-provided roots. Files never store +machine-local checkout paths. + +```sh +PARQUET_N6_GATE=1 \ +PARQUET_N6_FORMAT_ROOT=/path/to/parquet-format \ +PARQUET_N6_TESTING_ROOT=/path/to/parquet-testing \ +PARQUET_N6_JAVA_ROOT=/path/to/parquet-java \ +PARQUET_N6_ARROW_RS_ROOT=/path/to/arrow-rs \ +PARQUET_N6_ARROW_CPP_ROOT=/path/to/arrow-policy-source \ +PARQUET_N6_PYARROW_ROOT=/path/to/arrow-25-source \ +PARQUET_N6_DUCKDB_ROOT=/path/to/duckdb-1.5.5 \ +PARQUET_N6_JULIA_SOURCE_DEPOT=/path/to/clean-exact-julia-source-depot \ +PARQUET_N6_CPYTHON_312_ROOT=/path/to/cpython-3.12.8 \ +PARQUET_N6_CPYTHON_314_ROOT=/path/to/cpython-3.14.2 \ +PARQUET_N6_RAW_JAVA_DOWNLOAD_CACHE=/path/to/pinned-raw-java-downloads \ +PARQUET_N6_VALIDATOR_PYTHON_ARCHIVE=/path/to/cpython-3.14.2+20260127-aarch64-apple-darwin-install_only_stripped.tar.gz \ +PARQUET_N6_VALIDATOR_WHEEL_DIR=/path/to/pinned-validator-wheels \ +PARQUET_N6_INTEROP_PYTHON_ARCHIVE=/path/to/cpython-3.12.8+20250115-aarch64-apple-darwin-install_only_stripped.tar.gz \ +PARQUET_N6_INTEROP_WHEEL_DIR=/path/to/pinned-interop-wheels \ +PARQUET_N6_JAVA_JDK_ROOT=/path/to/temurin-21.0.8+9-jdk \ +PARQUET_N6_DOCKER=/absolute/path/to/docker \ +julia +1.12.6 --project=. --startup-file=no --history-file=no test/conformance/n6/runtests.jl +``` + +`PARQUET_N6_JULIA_SOURCE_DEPOT` must point to a clean exact package-source +depot. When set, it is the exclusive package and artifact source. It must +contain each descriptor-selected +`packages//` tree and each selected +`artifacts/` tree. Do not use a mutable working depot with +coverage files, preferences, compiled caches, or changed package sources. The +gate verifies each selected source tree before and after it copies the tree +into a fresh private depot. It locks that private depot read-only before Julia +loads Parquet.jl. + +The gate also verifies the exact Julia 1.12.6 runtime. It copies the complete +runtime into a fresh private directory and locks the copy read-only. The +Parquet.jl producer bootstrap and the independent model both use the Julia +executable from this same private runtime. The gate checks the source and +private runtime identities again after execution. + +The gate reads the exact bounded CPython 3.14.2 distribution archive once and +checks its pinned hash before extraction. It removes `site-packages` and all +bytecode caches, checks the clean tree, and locks the tree and exact wheel +snapshots read-only. +It starts the base interpreter with `-I -B -S` before it creates the fresh +validator environment. This prevents excluded system `site-packages`, `.pth` +files, and `sitecustomize` from running. It then starts the fresh environment +with `-I`. It does not trust an existing virtual environment or installed +package metadata. + +The interoperability gate verifies the exact CPython 3.12.8 standalone archive +before extraction. It removes the complete bundled `site-packages` tree and all +bytecode caches. It then verifies the portable clean-tree and executable hashes +before it runs any oracle with `-I -B -S`. + +The exact source gate always runs the raw scanner self-test and two scans of the +20-file corpus. It also regenerates and checks the normalized raw and +independent-model evidence. The raw scanner gate requires the exact eight-file +download cache named by `PARQUET_N6_RAW_JAVA_DOWNLOAD_CACHE`. It reads each +bounded file once, verifies the hash from `oracles/raw-java/toolchain.env`, and +copies the bytes into a fresh private build. The mandatory gate does not use the +network and does not write to the input cache. + +No file here authorizes an oracle image publication, a repository +`oracles.lock`, a frozen N5 evidence change, pruning, or a release claim. diff --git a/test/conformance/n6/artifacts.sha256 b/test/conformance/n6/artifacts.sha256 new file mode 100644 index 0000000..bf5a5ed --- /dev/null +++ b/test/conformance/n6/artifacts.sha256 @@ -0,0 +1,69 @@ +bec53a7c89094732bf5a5dd691a9239968a21748c4de9d7a9bcfee622ecf16af README.md +50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419 capabilities.toml +10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991 corpus-files.sha256 +5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31 evidence.schema.json +670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25 fixtures.toml +8b48613a795deed37089f6dbfca8eb6ed18062ebfd907a4606c91ef3308776ad harnesses/common.py +96e9219d8c5c0d24d348280dcbad3ee2006a5ec6f3096bb7ca671592ef422754 harnesses/duckdb.py +6b8d6f2bd1dfe4ce26a3f772974dd5a3b875c17972264563cee8cfec49b748f0 harnesses/pyarrow.py +f3d90b023c5ec3e71713d96631054a218e4656f2345fe407daeff60ece3ae694 harnesses/test_common.py +186cc635d7fcbc19f9acabc0ba9198d837d6eb0fccdd1a431a3137a6a43be565 julia/Manifest.toml +e3e0e903e3ecca103eeb5ec2d11aa0ee81c28f4344f48c8bfd1df0eaa5c01370 julia/N6ParquetJLHarness.jl +77dbc466d90fdc9e5795e33e2323dd6520696084849664519590c0e1d4af2857 julia/bootstrap.jl +508a7d640a7f24e6548f2973f6e2aa867834d1f91dd98b577ef40417d3209dd4 julia/generate.jl +4cf7759e22159ab3c6133bcecc3ba94b5857485581e586bf8dad49bc325a3a5e julia/parquet-jl-producer.toml +2a53ea964efe9e4a5e323c454a885a6406d8b5ff12edf096c1bff1727e74b266 julia/producer_gate.jl +82b188a7a5b36b43035b91a5dbd11a95501d5bd7eb33ec10eb1988b0c3add216 julia/runtests.jl +32c090ed6e0c6af49eabf3f96afc6e17dff630c4e89372367e693202e87c4262 model/N6StatisticsModel.jl +143290750312ae294c9954daf3d30d9d79b8627c67059ee001ba2228fa803814 model/README.md +925da4abeb448033b291e8f2a1ed8d68f6af30e5a4994d1e576f757581c5ad7b model/cases.toml +c74aab7a4d522a77443bba61070bae51c15bfec75a917bcd5ea16331d2307691 model/runtests.jl +82578bb410dad68d64a5e2f9ca65bff3b0850962a715552cf59a57617cb6b1f9 normalizers/README.md +749d8db3288b55cb42b7d739a7081b69a190905fc3f6e86c21182f7d421c2892 normalizers/check.sh +c18ac5ab62a8c90a0d8c3dea1b6678095a2c2aeb9cb1d495012122f105f8e261 normalizers/check_generated.sh +ea73b2731a981b8890baae47c756db2b42f0130926e100b91edace12fb2f0b0b normalizers/common.py +876e7382758b94c895e57769e8fb52216ac626394ff4026dee00f92812db7f3c normalizers/model-producer.toml +2468012b4536136c9737629842bb37424d65c82dbc0c0303b74d2612f69afb23 normalizers/model_bridge.jl +51932b834d4c38147dc3452c615f09d45cbbc03b6bf69e2327dcdf9508a63296 normalizers/normalize_model.py +fb9a8c0bcca7a4ca0a252e735a891120af441b8861d3321b57ba6adf9ac3099a normalizers/normalize_raw.py +53c941853a51b1f67d3b554b6032c5d47e2cab2d2839171d914486aa2e88045c normalizers/runtests.py +74fcbb6077f4f472dcfa22c31c380659b71fe03f1468d8c0e5823cbf7b5a69b6 oracles/arrow-rs/.gitignore +1925a94cd24d07fe1a34bef83644028a03dc209256787427f98df9f2090441b3 oracles/arrow-rs/README.md +7d2a79ea00d64ac50eea6e866a6ff94b20275e2bcdb27d7256b16070def316c7 oracles/arrow-rs/build.sh +453fc2b3a93d45c90f43bcb4684ed0726999529fb44dbc3f567db8fc416fa56d oracles/arrow-rs/check.sh +a7b9439da961d319d828c758badebb9f93d1d85625f2461e7867c34848d060cc oracles/arrow-rs/metadata/Cargo.toml +a2b50a6b20153dee9a4a8d48d8adae5f223b430909b90c6b208eaae3e067e9a8 oracles/arrow-rs/metadata/src/main.rs +992c487acf716a324cc49713c5ac1aefbc5a5b195bc66047830e9a16e0da833c oracles/arrow-rs/run.py +b386e4b83c599ee646beb72df02efcd5c61e77ef9b69e8fa1c4a0bc3fb1df084 oracles/arrow-rs/run.sh +a3857a242c9bad0f3a3405fd6222389a7f66b0726a547e7a24a470975e040c43 oracles/arrow-rs/runtests.py +c141bde2eea6442bcd8b3c7ad9f8aeadb04e7ad1080f5dc7b7cba0a2dae4cfe5 oracles/arrow-rs/toolchain.toml +309ef3d2c15a0c40b562f59bfc054db87342c55d6c926cc639112c5d8c12303f oracles/parquet-java/.gitignore +35c80bda052139afb2a3dc9416fd10807d42a72ae40b0c0b480fa78ceca63d0f oracles/parquet-java/README.md +26023ade8fc0bd16433c2ac91ae8a64285e7c8fac68f8f818a7e788a63859973 oracles/parquet-java/build.sh +1b7a43e1d9c57784551fbc9decad07581a0cd9282a09939b457007cf83ad65e3 oracles/parquet-java/check.sh +8cd6aa7e8efadf11a916e549330a6b3bd44a025d53f8cc55c3ecc82c15dcc85d oracles/parquet-java/run.py +68e2573a1636ca5424f3418f80221bd8af4fc660a985d58acbc1c464f197cf92 oracles/parquet-java/run.sh +9f671343988d078752140f231cb81a8ea01b24fa164cb0a3671770e13361cd0d oracles/parquet-java/runtests.py +8be45f063510bbd04f95115e8572eb4cff40914b35f5f2334452edb3f423e60a oracles/parquet-java/src/org/julialang/parquet/n6/java/AuditMain.java +a4459b555778b1979de10510fd5291372ffb9f41166539868ce3639494d5a452 oracles/parquet-java/toolchain.toml +f054861d112e67f8b98d353b139fcc790a06a4b4b31af6f3b6e106cf1014e4ce oracles/raw-java/.gitignore +1b1f4140c562ba73cd4efc500f8345d38e00b48998d1e431560ce6d786ee8a0d oracles/raw-java/README.md +e6a9c2ef7d0d5f8152a4e996fb5a35d2ef6cb5bcb75406024c185dcffe79b8fe oracles/raw-java/check.sh +8fb8144d6838906141f827736785540ac9ceaf638d60399c8d4c1940e34d8f28 oracles/raw-java/evidence.schema.json +e90f866346556f857c147512066d1b349c4546cb74164271826e9955c97b3449 oracles/raw-java/expected/self-test.jsonl +d595de66a27187223eb987765fc6c9c341d509ecd15de704656a2980bc6217bc oracles/raw-java/jdk-darwin-arm64-sequoia.manifest +b8d1358f167483f3edd98cb88d58599d05f5fab7d429259bca3ac0de30300229 oracles/raw-java/run.sh +d7d473707f8ef04da1e1635644792eb22a47438f23d1f22f4005f6d98d3c808a oracles/raw-java/scripts/build.sh +8cd5a66aef1536e03f5d2db976775ee508764619be42bae06d885eb8139e8b82 oracles/raw-java/scripts/common.sh +ac3a5efed475dd5613fb14ad43e84f628c4726381d85a757e1621dfdf3da4d85 oracles/raw-java/scripts/fetch-toolchain.sh +c08c11e0fa667cb1457dbe02852c76a0a180c58fb5caa795723e010d7f8ad2ae oracles/raw-java/scripts/generate.sh +42e3e1ad593eaa145147764ded64f7eef6aa2fbf9c9ece3a9948b6a0c145cbbd oracles/raw-java/src/org/julialang/parquet/n6/raw/CompactThriftPreflight.java +7dd4d65e650b5ba320fbb3d0951ba8f64afdb710b8c1801001b48bf1060cec92 oracles/raw-java/src/org/julialang/parquet/n6/raw/JsonWriter.java +923ac040755c58eef858e5f23b10fcc2b8b95b94e3b7039cded5701e718ee43e oracles/raw-java/src/org/julialang/parquet/n6/raw/RawFooterScanner.java +c943e362614da1f0f356a45a11a97f6af5e64ae99c28df13bc004a038693d94b oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestFixture.java +1905a501a671879ad683de58f9676c078ec19214bd1e22e91d70d0c1278557e9 oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestMain.java +8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629 oracles/raw-java/toolchain.env +b0708ac70a942093a631e849a2442169fd64376908b5e2b979055cb51fb7a3eb runtests.jl +a3c20486eae1ec54266ca94e9a29f8fa0c7e46bdddbb829d28bc92991ddda6ae toolchains/duckdb.toml +0e1e7fa951f82f12ddfdbeb3865936763c1818f1f20a655d2646b1186cbcd958 toolchains/pyarrow.toml +a13087f64ff74ede072acca1303deb3d7f2b3b03029ce1f668f4e8419b013ed3 validate_evidence.py diff --git a/test/conformance/n6/capabilities.toml b/test/conformance/n6/capabilities.toml new file mode 100644 index 0000000..2cb09b6 --- /dev/null +++ b/test/conformance/n6/capabilities.toml @@ -0,0 +1,523 @@ +matrix_version = 2 +plan_sha256 = "15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304" +fixture_manifest = "fixtures.toml" +unsupported_is_pass = false +statuses = ["verified", "planned", "unsupported", "not_assessed"] + +[[capability]] +id = "wire.column-order.type" +kind = "wire" + +[[capability]] +id = "wire.column-order.ieee" +kind = "wire" + +[[capability]] +id = "wire.column-order.empty" +kind = "wire" + +[[capability]] +id = "wire.statistics.deprecated-bounds" +kind = "wire" + +[[capability]] +id = "wire.statistics.modern-bounds" +kind = "wire" + +[[capability]] +id = "wire.statistics.exactness" +kind = "wire" + +[[capability]] +id = "wire.statistics.counts" +kind = "wire" + +[[capability]] +id = "wire.statistics.nan-count" +kind = "wire" + +[[capability]] +id = "semantic.type-order" +kind = "semantic" + +[[capability]] +id = "semantic.ieee-total-order" +kind = "semantic" + +[[capability]] +id = "semantic.logical-order" +kind = "semantic" + +[[capability]] +id = "semantic.count-state" +kind = "semantic" + +[[capability]] +id = "semantic.producer-trust" +kind = "semantic" + +[[capability]] +id = "read.logical-values" +kind = "runtime" + +[[capability]] +id = "read.statistics-metadata" +kind = "runtime" + +[[capability]] +id = "read.parquet-metadata-view" +kind = "runtime" + +[[capability]] +id = "write.type-order" +kind = "runtime" + +[[capability]] +id = "write.statistics-disabled" +kind = "runtime" + +[[capability]] +id = "write.statistics-limit" +kind = "runtime" + +[[capability]] +id = "read.no-pruning" +kind = "runtime" + +[[capability]] +id = "compat.legacy-statistics" +kind = "compatibility" + +[[authority]] +id = "parquet-format" +kind = "normative-specification" +version = "2.13.0" +revision = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" +platforms = ["source"] +toolchain_sha256 = [] + +[[authority.claim]] +capability = "semantic.type-order" +status = "verified" +cases = ["atomic-bound-family", "plain-bound-decoding"] +scope = "Pinned IDL and LogicalTypes.md rules only." + +[[authority.claim]] +capability = "semantic.ieee-total-order" +status = "verified" +cases = ["ieee-total-order", "float16-exhaustive"] +scope = "Pinned Parquet 2.13 ColumnOrder member and statistics rules only." + +[[authority.claim]] +capability = "wire.statistics.nan-count" +status = "verified" +cases = ["count-state-machine", "ieee-total-order"] +scope = "Pinned field 9 definition only; no runtime claim." + +[[authority]] +id = "parquet-testing" +kind = "apache-corpus" +version = "snapshot" +revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +platforms = ["data"] +toolchain_sha256 = [] + +[[authority.claim]] +capability = "wire.statistics.modern-bounds" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-bson", "apache-fixed-length-byte-array", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-floating-orders-nan-count", "apache-int32-with-null-pages", "apache-int96-timestamp-order", "apache-json", "apache-nan-in-stats", "apache-rle-boolean-encoding"] +scope = "Only the exact files in fixtures.toml." + +[[authority.claim]] +capability = "wire.statistics.deprecated-bounds" +status = "verified" +cases = ["apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-floating-orders-nan-count", "apache-int32-decimal", "apache-int32-with-null-pages", "apache-int64-decimal", "apache-nan-in-stats", "apache-rle-boolean-encoding"] +scope = "Only the exact files in fixtures.toml." + +[[authority.claim]] +capability = "wire.statistics.exactness" +status = "verified" +cases = ["apache-binary-truncated-min-max"] +scope = "Modern bound exactness fields in one pinned fixture." + +[[authority.claim]] +capability = "wire.statistics.nan-count" +status = "verified" +cases = ["apache-floating-orders-nan-count"] +scope = "Field 9 presence in one pinned fixture." + +[[authority.claim]] +capability = "wire.column-order.ieee" +status = "verified" +cases = ["apache-floating-orders-nan-count"] +scope = "IEEE order member presence in one pinned fixture." + +[[authority]] +id = "n6-independent-model" +kind = "test-owned-independent-semantics" +version = "1" +revision = "876e7382758b94c895e57769e8fb52216ac626394ff4026dee00f92812db7f3c" +platforms = ["julia-1.10", "julia-1.12"] +toolchain_sha256 = ["6f687953e48958fc6596962379691d1c8a1720d3a9ff39c1e3113888e43bd8ae", "c784c03af8ab52e48aa6f57ce8aa06a3c9160671054301ab6c3dfebf2e582525", "9ad38bea81ecace044a4bdef2a0246dee94cb8a44c9420809cc00f9872651c64", "273ec71de498a36c77a7e4bb3af4a3f75c338bd1cfe255cab30805b6a2cda76e"] + +[[authority.claim]] +capability = "semantic.type-order" +status = "verified" +cases = ["plain-bound-decoding", "atomic-bound-family", "type-order-float"] +scope = "Independent metadata-like values; no Parquet.jl import." + +[[authority.claim]] +capability = "semantic.ieee-total-order" +status = "verified" +cases = ["ieee-total-order", "float16-exhaustive", "independent-extrema"] +scope = "Raw-bit Float16, Float32, and Float64 model." + +[[authority.claim]] +capability = "semantic.logical-order" +status = "verified" +cases = ["logical-bound-validity", "decimal-order"] +scope = "Already validated leaf schemas only." + +[[authority.claim]] +capability = "semantic.count-state" +status = "verified" +cases = ["count-state-machine", "atomic-bound-family"] +scope = "Independent count and bound-family state machine." + +[[authority.claim]] +capability = "semantic.producer-trust" +status = "verified" +cases = ["producer-parquet-251", "producer-old-order"] +scope = "Pinned parquet-java and Arrow C++ policies." + +[[authority.claim]] +capability = "semantic.logical-order" +status = "verified" +cases = ["apache-binary", "apache-bson", "apache-byte-array-decimal", "apache-fixed-length-byte-array", "apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-int32-decimal", "apache-int64-decimal", "apache-json"] +scope = "Frozen combination with normalized raw leaf metadata." + +[[authority.claim]] +capability = "semantic.type-order" +status = "verified" +cases = ["apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-floating-orders-nan-count", "apache-int32-with-null-pages", "apache-nan-in-stats", "apache-rle-boolean-encoding"] +scope = "Frozen combination with normalized raw statistics." + +[[authority.claim]] +capability = "semantic.ieee-total-order" +status = "verified" +cases = ["apache-floating-orders-nan-count"] +scope = "Frozen combination with normalized raw IEEE evidence." + +[[authority.claim]] +capability = "semantic.count-state" +status = "verified" +cases = ["apache-floating-orders-nan-count", "apache-int32-with-null-pages", "apache-single-nan"] +scope = "Frozen combination with normalized raw counts." + +[[authority]] +id = "n6-raw-java" +kind = "test-owned-independent-wire-scanner" +version = "parquet-2.13-raw-footer-v3" +revision = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" +platforms = ["macos-15-arm64"] +toolchain_sha256 = ["8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629"] + +[[authority.claim]] +capability = "wire.column-order.type" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-bson", "apache-fixed-length-byte-array", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-floating-orders-nan-count", "apache-int32-with-null-pages", "apache-json", "apache-nan-in-stats", "apache-single-nan"] +scope = "Raw Compact-Thrift field headers in the 20 pinned fixtures." + +[[authority.claim]] +capability = "wire.column-order.ieee" +status = "verified" +cases = ["apache-floating-orders-nan-count"] +scope = "Raw Compact-Thrift field 2 in one pinned fixture." + +[[authority.claim]] +capability = "wire.column-order.empty" +status = "verified" +cases = ["apache-int96-timestamp-order"] +scope = "Empty ColumnOrder union in one pinned fixture." + +[[authority.claim]] +capability = "wire.statistics.deprecated-bounds" +status = "verified" +cases = ["apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-floating-orders-nan-count", "apache-int32-decimal", "apache-int32-with-null-pages", "apache-int64-decimal", "apache-nan-in-stats", "apache-rle-boolean-encoding"] +scope = "Raw Statistics fields 1 and 2 in the exact Apache fixtures." + +[[authority.claim]] +capability = "wire.statistics.modern-bounds" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-bson", "apache-fixed-length-byte-array", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-floating-orders-nan-count", "apache-int32-with-null-pages", "apache-int96-timestamp-order", "apache-json", "apache-nan-in-stats", "apache-rle-boolean-encoding"] +scope = "Raw Statistics fields 5 and 6 in the exact Apache fixtures." + +[[authority.claim]] +capability = "wire.statistics.exactness" +status = "verified" +cases = ["apache-binary-truncated-min-max"] +scope = "Raw Statistics fields 7 and 8 in one exact Apache fixture." + +[[authority.claim]] +capability = "wire.statistics.counts" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-bson", "apache-fixed-length-byte-array", "apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-floating-orders-nan-count", "apache-int32-decimal", "apache-int32-with-null-pages", "apache-int64-decimal", "apache-int96-timestamp-order", "apache-json", "apache-nan-in-stats", "apache-rle-boolean-encoding", "apache-single-nan"] +scope = "Raw Statistics fields 3, 4, and 9 in the exact fixtures." + +[[authority.claim]] +capability = "wire.statistics.nan-count" +status = "verified" +cases = ["apache-floating-orders-nan-count"] +scope = "Raw Statistics field 9 in one exact Apache fixture." + +[[authority.claim]] +capability = "semantic.type-order" +status = "unsupported" +cases = ["atomic-bound-family"] +scope = "The scanner records wire facts and applies no comparator semantics." + +[[authority.claim]] +capability = "wire.column-order.type" +status = "verified" +cases = ["julia-writer-type-order", "julia-writer-undefined-order", "julia-writer-oversized-bounds", "julia-writer-nested-row-groups"] +scope = "Raw validation of the exact Julia-writer fixtures." + +[[authority.claim]] +capability = "wire.column-order.ieee" +status = "verified" +cases = ["julia-writer-ieee-order"] +scope = "Raw validation of the exact Julia floating writer output." + +[[authority.claim]] +capability = "wire.statistics.nan-count" +status = "verified" +cases = ["julia-writer-ieee-order"] +scope = "Raw field 9 validation of the exact Julia floating writer output." + +[[authority.claim]] +capability = "wire.statistics.modern-bounds" +status = "verified" +cases = ["julia-writer-type-order", "julia-writer-ieee-order", "julia-writer-nested-row-groups"] +scope = "Raw modern bounds in the exact Julia-writer fixtures." + +[[authority.claim]] +capability = "wire.statistics.exactness" +status = "verified" +cases = ["julia-writer-type-order", "julia-writer-ieee-order", "julia-writer-nested-row-groups"] +scope = "Raw modern exactness fields in the exact Julia-writer fixtures." + +[[authority.claim]] +capability = "wire.statistics.counts" +status = "verified" +cases = ["julia-writer-type-order", "julia-writer-ieee-order", "julia-writer-undefined-order", "julia-writer-oversized-bounds", "julia-writer-nested-row-groups"] +scope = "Raw counts in the exact Julia-writer fixtures." + +[[authority]] +id = "parquet-jl" +kind = "implementation-under-test" +version = "1.0.0-DEV" +revision = "ea75000a8b4505c73efe50476a45dfe427ed5c8f7123fbaf244390f2b26b0c80" +platforms = ["julia-1.10", "julia-1.12"] +toolchain_sha256 = ["6f687953e48958fc6596962379691d1c8a1720d3a9ff39c1e3113888e43bd8ae", "9ad38bea81ecace044a4bdef2a0246dee94cb8a44c9420809cc00f9872651c64", "4cf7759e22159ab3c6133bcecc3ba94b5857485581e586bf8dad49bc325a3a5e"] + +[[authority.claim]] +capability = "write.type-order" +status = "verified" +cases = ["julia-writer-type-order", "julia-writer-ieee-order", "julia-writer-undefined-order", "julia-writer-oversized-bounds", "julia-writer-nested-row-groups"] +scope = "Exact N6 writer fixtures produced by the implementation under test." + +[[authority.claim]] +capability = "write.statistics-disabled" +status = "verified" +cases = ["julia-writer-statistics-disabled"] +scope = "No statistics or column_orders when statistics=false." + +[[authority.claim]] +capability = "write.statistics-limit" +status = "verified" +cases = ["julia-writer-oversized-bounds"] +scope = "Counts remain while both oversized bounds are omitted." + +[[authority.claim]] +capability = "semantic.type-order" +status = "verified" +cases = ["julia-writer-type-order", "julia-writer-undefined-order", "julia-writer-nested-row-groups"] +scope = "Production interpretation must agree with the independent model." + +[[authority.claim]] +capability = "semantic.logical-order" +status = "verified" +cases = ["julia-writer-type-order"] +scope = "Stable logical leaf orders must agree with the independent model." + +[[authority.claim]] +capability = "semantic.ieee-total-order" +status = "verified" +cases = ["julia-writer-ieee-order"] +scope = "Production floating extrema must agree with the independent model." + +[[authority.claim]] +capability = "semantic.count-state" +status = "verified" +cases = ["julia-writer-ieee-order", "julia-writer-nested-row-groups"] +scope = "Nested leaf-entry and dense-value counts." + +[[authority.claim]] +capability = "semantic.producer-trust" +status = "verified" +cases = ["julia-reader-untrusted-producer"] +scope = "Affected bounds become unknown without erasing valid counts." + +[[authority.claim]] +capability = "compat.legacy-statistics" +status = "verified" +cases = ["julia-reader-untrusted-producer"] +scope = "Modern and deprecated producer-policy families remain separate." + +[[authority.claim]] +capability = "read.no-pruning" +status = "verified" +cases = ["julia-reader-no-pruning-absent", "julia-reader-no-pruning-trusted", "julia-reader-no-pruning-untrusted", "julia-reader-no-pruning-oversized", "julia-reader-no-pruning-invalid"] +scope = "N6-A reads identical bodies and values for every statistics trust state." + +[[authority.claim]] +capability = "read.logical-values" +status = "verified" +cases = ["julia-writer-type-order", "julia-writer-ieee-order", "julia-writer-statistics-disabled", "julia-writer-nested-row-groups", "julia-reader-untrusted-producer", "julia-reader-no-pruning-absent", "julia-reader-no-pruning-trusted", "julia-reader-no-pruning-untrusted", "julia-reader-no-pruning-oversized", "julia-reader-no-pruning-invalid"] +scope = "Julia reads every generated N6 fixture without value changes." + +[[authority]] +id = "parquet-java" +kind = "external-implementation" +version = "1.17.1" +revision = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +platforms = ["macos-15-arm64"] +toolchain_sha256 = ["a4459b555778b1979de10510fd5291372ffb9f41166539868ce3639494d5a452"] + +[[authority.claim]] +capability = "read.logical-values" +status = "verified" +cases = ["apache-alltypes-dictionary", "apache-alltypes-plain", "apache-binary", "apache-bson", "apache-byte-array-decimal", "apache-fixed-length-byte-array", "apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-int32-decimal", "apache-int32-with-null-pages", "apache-int64-decimal", "apache-json", "apache-rle-boolean-encoding"] +scope = "Only listed N6 fixtures and values supported by the high-level API." + +[[authority.claim]] +capability = "wire.column-order.type" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-bson", "apache-fixed-length-byte-array", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-int32-with-null-pages", "apache-json", "apache-nan-in-stats", "apache-single-nan"] +scope = "Pinned Parquet Java metadata API independently reports TYPE_ORDER for every listed leaf." + +[[authority.claim]] +capability = "compat.legacy-statistics" +status = "verified" +cases = ["apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-int32-decimal", "apache-int64-decimal", "producer-parquet-251"] +scope = "Pinned high-level API and producer-version policy." + +[[authority.claim]] +capability = "wire.statistics.nan-count" +status = "unsupported" +cases = ["apache-floating-orders-nan-count"] +scope = "Embedded Parquet format 2.12 does not expose field 9." + +[[authority]] +id = "arrow-rs" +kind = "external-implementation" +version = "59.2.0" +revision = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +platforms = ["macos-15-arm64", "linux-amd64-offline-harness"] +toolchain_sha256 = ["c141bde2eea6442bcd8b3c7ad9f8aeadb04e7ad1080f5dc7b7cba0a2dae4cfe5"] + +[[authority.claim]] +capability = "read.logical-values" +status = "verified" +cases = ["apache-alltypes-dictionary", "apache-alltypes-plain", "apache-binary", "apache-bson", "apache-byte-array-decimal", "apache-fixed-length-byte-array", "apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-int32-decimal", "apache-int32-with-null-pages", "apache-int64-decimal", "apache-json", "apache-rle-boolean-encoding"] +scope = "Only listed N6 fixtures and values supported by Arrow Rust 59.2.0." + +[[authority.claim]] +capability = "wire.column-order.type" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-bson", "apache-fixed-length-byte-array", "apache-float16-nonzeros-and-nans", "apache-float16-zeros-and-nans", "apache-int32-with-null-pages", "apache-json", "apache-nan-in-stats", "apache-single-nan"] +scope = "Pinned Arrow Rust metadata API independently reports TYPE_ORDER for every listed leaf." + +[[authority.claim]] +capability = "wire.column-order.ieee" +status = "unsupported" +cases = ["apache-floating-orders-nan-count"] +scope = "Pinned metadata layer treats the future union member as unknown." + +[[authority.claim]] +capability = "wire.statistics.nan-count" +status = "unsupported" +cases = ["apache-floating-orders-nan-count"] +scope = "Pinned metadata layer does not expose field 9." + +[[authority]] +id = "apache-arrow-cpp-policy" +kind = "external-compatibility-policy" +version = "source" +revision = "515410b2a14ac766258e00b07eab9e5ee2692a62" +platforms = ["source"] +toolchain_sha256 = [] + +[[authority.claim]] +capability = "semantic.producer-trust" +status = "verified" +cases = ["producer-old-order"] +scope = "Pinned metadata.cc cutoff policy only." + +[[authority]] +id = "pyarrow" +kind = "external-implementation" +version = "25.0.1" +revision = "beccec0d0c451b7aa3e4530416ac431b3c035c69" +platforms = ["cp312-macosx-12-arm64"] +toolchain_sha256 = ["0e1e7fa951f82f12ddfdbeb3865936763c1818f1f20a655d2646b1186cbcd958"] + +[[authority.claim]] +capability = "read.logical-values" +status = "verified" +cases = ["apache-alltypes-dictionary", "apache-alltypes-plain", "apache-binary", "apache-bson", "apache-byte-array-decimal", "apache-fixed-length-byte-array", "apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-int32-decimal", "apache-int32-with-null-pages", "apache-int64-decimal", "apache-json", "apache-rle-boolean-encoding"] +scope = "Exact CPython 3.12 macOS arm64 wheel and listed fixtures only." + +[[authority.claim]] +capability = "read.statistics-metadata" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-bson", "apache-fixed-length-byte-array", "apache-int32-with-null-pages", "apache-json", "apache-nan-in-stats"] +scope = "Only fields exposed by PyArrow's row-group statistics API." + +[[authority.claim]] +capability = "wire.statistics.nan-count" +status = "unsupported" +cases = ["apache-floating-orders-nan-count"] +scope = "The high-level API does not expose raw field 9." + +[[authority]] +id = "duckdb" +kind = "external-implementation" +version = "1.5.5" +revision = "d8cdaa33fda8df955cc76ef58a280f68f4cd43fa" +platforms = ["cp312-macosx-11-arm64"] +toolchain_sha256 = ["a3c20486eae1ec54266ca94e9a29f8fa0c7e46bdddbb829d28bc92991ddda6ae"] + +[[authority.claim]] +capability = "read.logical-values" +status = "verified" +cases = ["apache-alltypes-dictionary", "apache-alltypes-plain", "apache-binary", "apache-byte-array-decimal", "apache-fixed-length-byte-array", "apache-fixed-length-decimal", "apache-fixed-length-decimal-legacy", "apache-int32-decimal", "apache-int32-with-null-pages", "apache-int64-decimal", "apache-json", "apache-rle-boolean-encoding"] +scope = "Exact CPython 3.12 macOS arm64 wheel and listed fixtures only." + +[[authority.claim]] +capability = "read.parquet-metadata-view" +status = "verified" +cases = ["apache-binary", "apache-binary-truncated-min-max", "apache-fixed-length-byte-array", "apache-int32-with-null-pages", "apache-json", "apache-nan-in-stats"] +scope = "Only fields exposed by parquet_metadata for listed fixtures." + +[[authority.claim]] +capability = "read.parquet-metadata-view" +status = "unsupported" +cases = ["apache-bson"] +scope = "DuckDB 1.5.5 rejects BSON converted type 20 in parquet_metadata." + +[[authority.claim]] +capability = "wire.column-order.ieee" +status = "not_assessed" +cases = ["apache-floating-orders-nan-count"] +scope = "No raw Compact-Thrift capability is claimed." diff --git a/test/conformance/n6/corpus-files.sha256 b/test/conformance/n6/corpus-files.sha256 new file mode 100644 index 0000000..cba25b1 --- /dev/null +++ b/test/conformance/n6/corpus-files.sha256 @@ -0,0 +1,20 @@ +7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb data/alltypes_dictionary.parquet +12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4 data/alltypes_plain.parquet +b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b data/binary.parquet +94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201 data/binary_truncated_min_max.parquet +44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129 data/bson.parquet +9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d data/byte_array_decimal.parquet +a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7 data/fixed_length_byte_array.parquet +67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2 data/fixed_length_decimal.parquet +323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a data/fixed_length_decimal_legacy.parquet +d0117dd9655992b869f8207235526a7d8931e079fd68d68c88c0170faa1f11ee data/float16_nonzeros_and_nans.parquet +4901850e7dcd64588a49391fa1dddca514b598e6f4266239033ea73513850f47 data/float16_zeros_and_nans.parquet +17f7d7655a089b9504a828dffaccd72225a9a6fd2a697099b5336ab274386f0a data/floating_orders_nan_count.parquet +3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9 data/int32_decimal.parquet +392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7 data/int32_with_null_pages.parquet +e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56 data/int64_decimal.parquet +e35f8748d286a729e719a01c5411f81c79802d61a55046d5cf9a663918a14644 data/int96_timestamp_order.parquet +594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5 data/json.parquet +77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337 data/nan_in_stats.parquet +585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76 data/rle_boolean_encoding.parquet +ea3371c44ed1794843a2f529888120537f68aedcb80d6fbe32cea1003ab5769e data/single_nan.parquet diff --git a/test/conformance/n6/evidence.schema.json b/test/conformance/n6/evidence.schema.json new file mode 100644 index 0000000..a4eeca8 --- /dev/null +++ b/test/conformance/n6/evidence.schema.json @@ -0,0 +1,394 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://julialang.github.io/Parquet.jl/conformance/n6/evidence.schema.json", + "title": "Parquet.jl N6 normalized statistics evidence", + "oneOf": [ + { "$ref": "#/$defs/run" }, + { "$ref": "#/$defs/file" }, + { "$ref": "#/$defs/columnStatistics" }, + { "$ref": "#/$defs/caseResult" } + ], + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "caseId": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)*$" + }, + "capabilityId": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[._-][a-z0-9]+)+$" + }, + "relativePath": { + "type": "string", + "pattern": "^(?!/)(?!.*(?:^|/)\\.{1,2}(?:/|$))(?!.*\\\\)[A-Za-z0-9._+@=-]+(?:/[A-Za-z0-9._+@=-]+)*$" + }, + "hexBytes": { + "type": ["string", "null"], + "pattern": "^(?:[0-9a-f]{2})*$" + }, + "canonicalInt64": { + "type": "string", + "pattern": "^(?:0|-?[1-9][0-9]*)$" + }, + "nullableInt64": { + "oneOf": [ + { "$ref": "#/$defs/canonicalInt64" }, + { "type": "null" } + ] + }, + "physicalType": { + "enum": [ + "BOOLEAN", + "INT32", + "INT64", + "INT96", + "FLOAT", + "DOUBLE", + "BYTE_ARRAY", + "FIXED_LEN_BYTE_ARRAY" + ] + }, + "logicalType": { + "enum": [ + "NONE", + "STRING", + "MAP", + "LIST", + "ENUM", + "DECIMAL", + "DATE", + "TIME", + "TIMESTAMP", + "INTEGER", + "UNKNOWN", + "JSON", + "BSON", + "UUID", + "FLOAT16", + "INTERVAL", + "VARIANT", + "GEOMETRY", + "GEOGRAPHY" + ] + }, + "convertedType": { + "enum": [ + "UTF8", + "MAP", + "MAP_KEY_VALUE", + "LIST", + "ENUM", + "DECIMAL", + "DATE", + "TIME_MILLIS", + "TIME_MICROS", + "TIMESTAMP_MILLIS", + "TIMESTAMP_MICROS", + "UINT_8", + "UINT_16", + "UINT_32", + "UINT_64", + "INT_8", + "INT_16", + "INT_32", + "INT_64", + "JSON", + "BSON", + "INTERVAL", + null + ] + }, + "leafSchema": { + "type": "object", + "additionalProperties": false, + "required": [ + "physical_type", + "logical_type", + "converted_type", + "type_length", + "precision", + "scale", + "bit_width", + "is_signed", + "time_unit", + "is_adjusted_to_utc", + "crs", + "geography_algorithm" + ], + "properties": { + "physical_type": { "$ref": "#/$defs/physicalType" }, + "logical_type": { "$ref": "#/$defs/logicalType" }, + "converted_type": { "$ref": "#/$defs/convertedType" }, + "type_length": { + "type": ["integer", "null"], + "minimum": -2147483648, + "maximum": 2147483647 + }, + "precision": { "type": ["integer", "null"], "minimum": 1 }, + "scale": { "type": ["integer", "null"], "minimum": 0 }, + "bit_width": { "enum": [8, 16, 32, 64, null] }, + "is_signed": { "type": ["boolean", "null"] }, + "time_unit": { "enum": ["MILLIS", "MICROS", "NANOS", null] }, + "is_adjusted_to_utc": { "type": ["boolean", "null"] }, + "crs": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { "type": "null" } + ] + }, + "geography_algorithm": { + "enum": ["SPHERICAL", "VINCENTY", "THOMAS", "ANDOYER", "KARNEY", null] + } + } + }, + "columnOrder": { + "type": "object", + "additionalProperties": false, + "required": ["state", "field_id", "wire_type", "header_hex"], + "properties": { + "state": { + "enum": [ + "ABSENT", + "TYPE_ORDER", + "IEEE_754_TOTAL_ORDER", + "UNKNOWN", + "WRONG_TYPE", + "EMPTY" + ] + }, + "field_id": { + "oneOf": [ + { "type": "integer", "minimum": -32768, "maximum": 32767 }, + { "type": "null" } + ] + }, + "wire_type": { + "oneOf": [ + { "type": "integer", "minimum": 0, "maximum": 16 }, + { "type": "null" } + ] + }, + "header_hex": { + "oneOf": [ + { "type": "string", "pattern": "^(?:[0-9a-f]{2})+$" }, + { "type": "null" } + ] + } + } + }, + "upstreamEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["evidence_id", "file", "sha256"], + "properties": { + "evidence_id": { "$ref": "#/$defs/caseId" }, + "file": { "$ref": "#/$defs/relativePath" }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "run": { + "type": "object", + "additionalProperties": false, + "required": [ + "record", + "schema_version", + "evidence_id", + "producer", + "producer_version", + "source_revision", + "plan_sha256", + "capabilities_sha256", + "fixture_manifest_sha256", + "corpus_manifest_sha256", + "evidence_schema_sha256", + "toolchain_sha256", + "unsupported_cases" + ], + "properties": { + "record": { "const": "run" }, + "schema_version": { "const": 2 }, + "evidence_id": { "$ref": "#/$defs/caseId" }, + "producer": { "type": "string", "minLength": 1 }, + "producer_version": { "type": "string", "minLength": 1 }, + "source_revision": { "type": "string", "minLength": 1 }, + "plan_sha256": { "$ref": "#/$defs/sha256" }, + "capabilities_sha256": { "$ref": "#/$defs/sha256" }, + "fixture_manifest_sha256": { "$ref": "#/$defs/sha256" }, + "corpus_manifest_sha256": { "$ref": "#/$defs/sha256" }, + "evidence_schema_sha256": { "$ref": "#/$defs/sha256" }, + "toolchain_sha256": { "$ref": "#/$defs/sha256" }, + "unsupported_cases": { + "type": "array", + "items": { "$ref": "#/$defs/caseId" }, + "uniqueItems": true + }, + "upstream_evidence": { + "type": "array", + "items": { "$ref": "#/$defs/upstreamEvidence" }, + "minItems": 1, + "uniqueItems": true + } + } + }, + "file": { + "type": "object", + "additionalProperties": false, + "required": [ + "record", + "schema_version", + "case_id", + "file", + "sha256", + "size", + "footer_length", + "row_group_count", + "leaf_count", + "column_order_count", + "created_by_present", + "created_by" + ], + "properties": { + "record": { "const": "file" }, + "schema_version": { "const": 2 }, + "case_id": { "$ref": "#/$defs/caseId" }, + "file": { "$ref": "#/$defs/relativePath" }, + "sha256": { "$ref": "#/$defs/sha256" }, + "size": { "type": "integer", "minimum": 12 }, + "footer_length": { "type": "integer", "minimum": 1 }, + "row_group_count": { "type": "integer", "minimum": 0 }, + "leaf_count": { "type": "integer", "minimum": 0 }, + "column_order_count": { "type": ["integer", "null"], "minimum": 0 }, + "created_by_present": { "type": "boolean" }, + "created_by": { "type": ["string", "null"] } + }, + "allOf": [ + { + "if": { "properties": { "created_by_present": { "const": false } } }, + "then": { "properties": { "created_by": { "type": "null" } } } + }, + { + "if": { "properties": { "created_by_present": { "const": true } } }, + "then": { "properties": { "created_by": { "type": "string" } } } + } + ] + }, + "columnStatistics": { + "type": "object", + "additionalProperties": false, + "required": [ + "record", + "schema_version", + "case_id", + "file", + "row_group", + "leaf", + "path", + "leaf_schema", + "column_order", + "num_values", + "has_statistics", + "deprecated_min_hex", + "deprecated_max_hex", + "min_value_hex", + "max_value_hex", + "is_min_value_exact", + "is_max_value_exact", + "null_count", + "distinct_count", + "nan_count", + "unknown_statistics_field_ids" + ], + "properties": { + "record": { "const": "column_statistics" }, + "schema_version": { "const": 2 }, + "case_id": { "$ref": "#/$defs/caseId" }, + "file": { "$ref": "#/$defs/relativePath" }, + "row_group": { "type": "integer", "minimum": 0 }, + "leaf": { "type": "integer", "minimum": 0 }, + "path": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "leaf_schema": { "$ref": "#/$defs/leafSchema" }, + "column_order": { "$ref": "#/$defs/columnOrder" }, + "num_values": { "$ref": "#/$defs/canonicalInt64" }, + "has_statistics": { "type": "boolean" }, + "deprecated_min_hex": { "$ref": "#/$defs/hexBytes" }, + "deprecated_max_hex": { "$ref": "#/$defs/hexBytes" }, + "min_value_hex": { "$ref": "#/$defs/hexBytes" }, + "max_value_hex": { "$ref": "#/$defs/hexBytes" }, + "is_min_value_exact": { "type": ["boolean", "null"] }, + "is_max_value_exact": { "type": ["boolean", "null"] }, + "null_count": { "$ref": "#/$defs/nullableInt64" }, + "distinct_count": { "$ref": "#/$defs/nullableInt64" }, + "nan_count": { "$ref": "#/$defs/nullableInt64" }, + "unknown_statistics_field_ids": { + "type": "array", + "items": { "type": "integer", "minimum": -32768, "maximum": 32767 }, + "uniqueItems": true + } + } + }, + "caseResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "record", + "schema_version", + "case_id", + "capability_id", + "digest_contract", + "status", + "expected_sha256", + "actual_sha256", + "detail" + ], + "properties": { + "record": { "const": "case_result" }, + "schema_version": { "const": 2 }, + "case_id": { "$ref": "#/$defs/caseId" }, + "capability_id": { "$ref": "#/$defs/capabilityId" }, + "digest_contract": { "$ref": "#/$defs/caseId" }, + "status": { "enum": ["PASS", "FAIL", "UNSUPPORTED"] }, + "expected_sha256": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "actual_sha256": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "detail": { "type": "string", "minLength": 1 } + }, + "allOf": [ + { + "if": { "properties": { "status": { "const": "PASS" } } }, + "then": { + "properties": { + "expected_sha256": { "$ref": "#/$defs/sha256" }, + "actual_sha256": { "$ref": "#/$defs/sha256" } + } + } + }, + { + "if": { "properties": { "status": { "const": "UNSUPPORTED" } } }, + "then": { + "properties": { + "expected_sha256": { "type": "null" }, + "actual_sha256": { "type": "null" }, + "detail": { "type": "string", "minLength": 1 } + } + } + } + ] + } + } +} diff --git a/test/conformance/n6/evidence/arrow-rs.normalized.jsonl b/test/conformance/n6/evidence/arrow-rs.normalized.jsonl new file mode 100644 index 0000000..1794528 --- /dev/null +++ b/test/conformance/n6/evidence/arrow-rs.normalized.jsonl @@ -0,0 +1,45 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-arrow-rs","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"arrow-rs","producer_version":"59.2.0","record":"run","schema_version":2,"source_revision":"782e5a685501a9db6cc8e9a3b7cbff894940c47a","toolchain_sha256":"c141bde2eea6442bcd8b3c7ad9f8aeadb04e7ad1080f5dc7b7cba0a2dae4cfe5","unsupported_cases":["apache-floating-orders-nan-count"],"upstream_evidence":[{"evidence_id":"normalized-raw-java-apache-corpus","file":"test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl","sha256":"1692d30284b57581993d524d41baa16b43e94b26dd3832bd6d63d689a1bb4ff8"}]} +{"case_id":"apache-alltypes-dictionary","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_dictionary.parquet","footer_length":723,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb","size":1698} +{"actual_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","capability_id":"read.logical-values","case_id":"apache-alltypes-dictionary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-alltypes-plain","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_plain.parquet","footer_length":730,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4","size":1851} +{"actual_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","capability_id":"read.logical-values","case_id":"apache-alltypes-plain","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build 031a6654009e3b82020012a18434c582bd74c73a)","created_by_present":true,"file":"data/binary.parquet","footer_length":371,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b","size":478} +{"actual_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","capability_id":"read.logical-values","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2","capability_id":"wire.column-order.type","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary-truncated-min-max","column_order_count":6,"created_by":"parquet-rs version 55.1.0","created_by_present":true,"file":"data/binary_truncated_min_max.parquet","footer_length":1358,"leaf_count":6,"record":"file","row_group_count":1,"schema_version":2,"sha256":"94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201","size":3070} +{"actual_sha256":"386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae","capability_id":"wire.column-order.type","case_id":"apache-binary-truncated-min-max","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-bson","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/bson.parquet","footer_length":280,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129","size":412} +{"actual_sha256":"eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a","capability_id":"read.logical-values","case_id":"apache-bson","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c","capability_id":"wire.column-order.type","case_id":"apache-bson","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-byte-array-decimal","column_order_count":null,"created_by":"HVR 5.3.0/9 (linux_glibc2.5-x64-64bit)","created_by_present":true,"file":"data/byte_array_decimal.parquet","footer_length":119,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d","size":324} +{"actual_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","capability_id":"read.logical-values","case_id":"apache-byte-array-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-byte-array","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build d057b39d93014fe40f5067ee4a33621e65c91552)","created_by_present":true,"file":"data/fixed_length_byte_array.parquet","footer_length":253,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7","size":4437} +{"actual_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","capability_id":"read.logical-values","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af","capability_id":"wire.column-order.type","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2","size":677} +{"actual_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal-legacy","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal_legacy.parquet","footer_length":336,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a","size":537} +{"actual_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal-legacy","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-nonzeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_nonzeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"d0117dd9655992b869f8207235526a7d8931e079fd68d68c88c0170faa1f11ee","size":501} +{"actual_sha256":"495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b","capability_id":"wire.column-order.type","case_id":"apache-float16-nonzeros-and-nans","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-zeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_zeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"4901850e7dcd64588a49391fa1dddca514b598e6f4266239033ea73513850f47","size":489} +{"actual_sha256":"1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665","capability_id":"wire.column-order.type","case_id":"apache-float16-zeros-and-nans","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-floating-orders-nan-count","column_order_count":6,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build c5dcd8ca5bad5fde9c797b876a16b5bf3b9206c0)","created_by_present":true,"file":"data/floating_orders_nan_count.parquet","footer_length":3026,"leaf_count":6,"record":"file","row_group_count":5,"schema_version":2,"sha256":"17f7d7655a089b9504a828dffaccd72225a9a6fd2a697099b5336ab274386f0a","size":6143} +{"actual_sha256":null,"capability_id":"wire.column-order.ieee","case_id":"apache-floating-orders-nan-count","detail":"The reviewed capability matrix marks this result unsupported.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":null,"record":"case_result","schema_version":2,"status":"UNSUPPORTED"} +{"actual_sha256":null,"capability_id":"wire.statistics.nan-count","case_id":"apache-floating-orders-nan-count","detail":"The reviewed capability matrix marks this result unsupported.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":null,"record":"case_result","schema_version":2,"status":"UNSUPPORTED"} +{"case_id":"apache-int32-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int32_decimal.parquet","footer_length":329,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9","size":478} +{"actual_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","capability_id":"read.logical-values","case_id":"apache-int32-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-with-null-pages","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build 433de8df33fcf31927f7b51456be9f53e64d48b9)","created_by_present":true,"file":"data/int32_with_null_pages.parquet","footer_length":265,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7","size":3829} +{"actual_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","capability_id":"read.logical-values","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a","capability_id":"wire.column-order.type","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int64-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int64_decimal.parquet","footer_length":338,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56","size":591} +{"actual_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","capability_id":"read.logical-values","case_id":"apache-int64-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-json","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/json.parquet","footer_length":270,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5","size":402} +{"actual_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","capability_id":"read.logical-values","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04","capability_id":"wire.column-order.type","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-nan-in-stats","column_order_count":1,"created_by":"parquet-cpp version 1.3.2-SNAPSHOT","created_by_present":true,"file":"data/nan_in_stats.parquet","footer_length":156,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337","size":329} +{"actual_sha256":"3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7","capability_id":"wire.column-order.type","case_id":"apache-nan-in-stats","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-rle-boolean-encoding","column_order_count":null,"created_by":null,"created_by_present":false,"file":"data/rle_boolean_encoding.parquet","footer_length":111,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76","size":192} +{"actual_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","capability_id":"read.logical-values","case_id":"apache-rle-boolean-encoding","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-single-nan","column_order_count":1,"created_by":"parquet-cpp version 1.5.1-SNAPSHOT","created_by_present":true,"file":"data/single_nan.parquet","footer_length":567,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"ea3371c44ed1794843a2f529888120537f68aedcb80d6fbe32cea1003ab5769e","size":660} +{"actual_sha256":"6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d","capability_id":"wire.column-order.type","case_id":"apache-single-nan","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d","record":"case_result","schema_version":2,"status":"PASS"} diff --git a/test/conformance/n6/evidence/duckdb.normalized.jsonl b/test/conformance/n6/evidence/duckdb.normalized.jsonl new file mode 100644 index 0000000..28dfbab --- /dev/null +++ b/test/conformance/n6/evidence/duckdb.normalized.jsonl @@ -0,0 +1,35 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-duckdb","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"duckdb","producer_version":"1.5.5","record":"run","schema_version":2,"source_revision":"d8cdaa33fda8df955cc76ef58a280f68f4cd43fa","toolchain_sha256":"a3c20486eae1ec54266ca94e9a29f8fa0c7e46bdddbb829d28bc92991ddda6ae","unsupported_cases":["apache-bson"],"upstream_evidence":[{"evidence_id":"normalized-raw-java-apache-corpus","file":"test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl","sha256":"1692d30284b57581993d524d41baa16b43e94b26dd3832bd6d63d689a1bb4ff8"}]} +{"case_id":"apache-alltypes-dictionary","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_dictionary.parquet","footer_length":723,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb","size":1698} +{"actual_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","capability_id":"read.logical-values","case_id":"apache-alltypes-dictionary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-alltypes-plain","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_plain.parquet","footer_length":730,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4","size":1851} +{"actual_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","capability_id":"read.logical-values","case_id":"apache-alltypes-plain","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build 031a6654009e3b82020012a18434c582bd74c73a)","created_by_present":true,"file":"data/binary.parquet","footer_length":371,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b","size":478} +{"actual_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","capability_id":"read.logical-values","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"1a70a48d47f90345fe7da65d5d018d82aec695d91d9018d9cd9184b655ea438f","capability_id":"read.parquet-metadata-view","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"1a70a48d47f90345fe7da65d5d018d82aec695d91d9018d9cd9184b655ea438f","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary-truncated-min-max","column_order_count":6,"created_by":"parquet-rs version 55.1.0","created_by_present":true,"file":"data/binary_truncated_min_max.parquet","footer_length":1358,"leaf_count":6,"record":"file","row_group_count":1,"schema_version":2,"sha256":"94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201","size":3070} +{"actual_sha256":"fd4b4b68f91cf9f80e72b805f6e3bfdba39384fe41c07c963e65c1d6295cdfe0","capability_id":"read.parquet-metadata-view","case_id":"apache-binary-truncated-min-max","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"fd4b4b68f91cf9f80e72b805f6e3bfdba39384fe41c07c963e65c1d6295cdfe0","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-bson","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/bson.parquet","footer_length":280,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129","size":412} +{"actual_sha256":null,"capability_id":"read.parquet-metadata-view","case_id":"apache-bson","detail":"The reviewed capability matrix marks this result unsupported.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":null,"record":"case_result","schema_version":2,"status":"UNSUPPORTED"} +{"case_id":"apache-byte-array-decimal","column_order_count":null,"created_by":"HVR 5.3.0/9 (linux_glibc2.5-x64-64bit)","created_by_present":true,"file":"data/byte_array_decimal.parquet","footer_length":119,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d","size":324} +{"actual_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","capability_id":"read.logical-values","case_id":"apache-byte-array-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-byte-array","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build d057b39d93014fe40f5067ee4a33621e65c91552)","created_by_present":true,"file":"data/fixed_length_byte_array.parquet","footer_length":253,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7","size":4437} +{"actual_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","capability_id":"read.logical-values","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"7a3bdbf51222a7516e4849e1ed60c683beb7bf92d0ba9cef78c9fcc8a597b7b5","capability_id":"read.parquet-metadata-view","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7a3bdbf51222a7516e4849e1ed60c683beb7bf92d0ba9cef78c9fcc8a597b7b5","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2","size":677} +{"actual_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal-legacy","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal_legacy.parquet","footer_length":336,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a","size":537} +{"actual_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal-legacy","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int32_decimal.parquet","footer_length":329,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9","size":478} +{"actual_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","capability_id":"read.logical-values","case_id":"apache-int32-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-with-null-pages","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build 433de8df33fcf31927f7b51456be9f53e64d48b9)","created_by_present":true,"file":"data/int32_with_null_pages.parquet","footer_length":265,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7","size":3829} +{"actual_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","capability_id":"read.logical-values","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"550a548f27483fbf56d26062b45c03c738a54b8e181b6cb515e6d2f669e11f67","capability_id":"read.parquet-metadata-view","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"550a548f27483fbf56d26062b45c03c738a54b8e181b6cb515e6d2f669e11f67","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int64-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int64_decimal.parquet","footer_length":338,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56","size":591} +{"actual_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","capability_id":"read.logical-values","case_id":"apache-int64-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-json","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/json.parquet","footer_length":270,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5","size":402} +{"actual_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","capability_id":"read.logical-values","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"4512003e839e774cb9bfcbb46d26cd8171c9aa75b03a094f0cae7ec219511b18","capability_id":"read.parquet-metadata-view","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"4512003e839e774cb9bfcbb46d26cd8171c9aa75b03a094f0cae7ec219511b18","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-nan-in-stats","column_order_count":1,"created_by":"parquet-cpp version 1.3.2-SNAPSHOT","created_by_present":true,"file":"data/nan_in_stats.parquet","footer_length":156,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337","size":329} +{"actual_sha256":"da4800a10319b9026804fe0b965c82956b11925f36d9aa4e447dcea16e4f3740","capability_id":"read.parquet-metadata-view","case_id":"apache-nan-in-stats","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"da4800a10319b9026804fe0b965c82956b11925f36d9aa4e447dcea16e4f3740","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-rle-boolean-encoding","column_order_count":null,"created_by":null,"created_by_present":false,"file":"data/rle_boolean_encoding.parquet","footer_length":111,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76","size":192} +{"actual_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","capability_id":"read.logical-values","case_id":"apache-rle-boolean-encoding","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","record":"case_result","schema_version":2,"status":"PASS"} diff --git a/test/conformance/n6/evidence/independent-model.normalized.jsonl b/test/conformance/n6/evidence/independent-model.normalized.jsonl new file mode 100644 index 0000000..65ed999 --- /dev/null +++ b/test/conformance/n6/evidence/independent-model.normalized.jsonl @@ -0,0 +1,128 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-independent-model","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"n6-independent-model","producer_version":"1","record":"run","schema_version":2,"source_revision":"876e7382758b94c895e57769e8fb52216ac626394ff4026dee00f92812db7f3c","toolchain_sha256":"273ec71de498a36c77a7e4bb3af4a3f75c338bd1cfe255cab30805b6a2cda76e","unsupported_cases":[],"upstream_evidence":[{"evidence_id":"normalized-raw-java-apache-corpus","file":"test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl","sha256":"1692d30284b57581993d524d41baa16b43e94b26dd3832bd6d63d689a1bb4ff8"}]} +{"actual_sha256":"ce7d5eba1d6c8167af6d8c78b753c76dda1adcbdce93427569bca80334ea054d","capability_id":"semantic.count-state","case_id":"atomic-bound-family","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ce7d5eba1d6c8167af6d8c78b753c76dda1adcbdce93427569bca80334ea054d","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"03356ecaa321e342f7362031499f4a110c6d426619b3abcea19bd869f6f38fb4","capability_id":"semantic.type-order","case_id":"atomic-bound-family","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"03356ecaa321e342f7362031499f4a110c6d426619b3abcea19bd869f6f38fb4","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"d8a7163f59defdcb8c771e07c0ddec65e873967340355e2eda07f63129eeff01","capability_id":"semantic.count-state","case_id":"count-state-machine","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d8a7163f59defdcb8c771e07c0ddec65e873967340355e2eda07f63129eeff01","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"b9fc8ce26cbbb5f44922aa90053520249f7d2e35a566740ebfd7d3436dfdaf1a","capability_id":"semantic.logical-order","case_id":"decimal-order","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b9fc8ce26cbbb5f44922aa90053520249f7d2e35a566740ebfd7d3436dfdaf1a","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"e4d2981184aa5082519114ba3dc7e5f000fdf0e70067fe33750dbe6ac7c6717b","capability_id":"semantic.ieee-total-order","case_id":"float16-exhaustive","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"e4d2981184aa5082519114ba3dc7e5f000fdf0e70067fe33750dbe6ac7c6717b","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"d596fee1637b66a94cef82ad91332ef69393fa486400ea8ee7cdbcde2ac97ccd","capability_id":"semantic.ieee-total-order","case_id":"ieee-total-order","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d596fee1637b66a94cef82ad91332ef69393fa486400ea8ee7cdbcde2ac97ccd","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f7e06eaadeda5d00e52ed4f7196cb3e39ce867fd256814bf8876ad165cfb1288","capability_id":"semantic.ieee-total-order","case_id":"independent-extrema","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f7e06eaadeda5d00e52ed4f7196cb3e39ce867fd256814bf8876ad165cfb1288","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"59acacd3241521be47618a40f279fb36bd4141e2c971f510551b6e7d8b34ed53","capability_id":"semantic.logical-order","case_id":"logical-bound-validity","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"59acacd3241521be47618a40f279fb36bd4141e2c971f510551b6e7d8b34ed53","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"15a1da9c96f3c7cf71dc3a61022aa49bc9b6c235f22288fd0340c460f728eee1","capability_id":"semantic.type-order","case_id":"plain-bound-decoding","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"15a1da9c96f3c7cf71dc3a61022aa49bc9b6c235f22288fd0340c460f728eee1","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"689dc5b2feee3b711beb804acd48d1c1b47f6d5654e94bccba1d7429154dd91d","capability_id":"semantic.producer-trust","case_id":"producer-old-order","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"689dc5b2feee3b711beb804acd48d1c1b47f6d5654e94bccba1d7429154dd91d","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"fb6f35682ebf84d6b3bb9a505cc91dd6154c8f5c05fab20798081afb11d1f496","capability_id":"semantic.producer-trust","case_id":"producer-parquet-251","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"fb6f35682ebf84d6b3bb9a505cc91dd6154c8f5c05fab20798081afb11d1f496","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"fb4ad657bd0f65b0e9b6b0c470e4290994724f60ef06774146c3d0b7777b7023","capability_id":"semantic.type-order","case_id":"type-order-float","detail":"The frozen independent model suite passed this semantic case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"fb4ad657bd0f65b0e9b6b0c470e4290994724f60ef06774146c3d0b7777b7023","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-alltypes-dictionary","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_dictionary.parquet","footer_length":723,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb","size":1698} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["id"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["bool_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["tinyint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["smallint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["int_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT64","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["bigint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["float_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["double_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":8,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["date_string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":9,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":10,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT96","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["timestamp_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_plain.parquet","footer_length":730,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4","size":1851} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["id"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["bool_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["tinyint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["smallint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["int_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT64","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["bigint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["float_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["double_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":8,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["date_string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":9,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":10,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT96","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["timestamp_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build 031a6654009e3b82020012a18434c582bd74c73a)","created_by_present":true,"file":"data/binary.parquet","footer_length":371,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b","size":478} +{"case_id":"apache-binary","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0b","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"12","path":["foo"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"d6fd29de2111abe686c631fd5ffa090530a21d247aa75cbd35ec237ae626e33b","capability_id":"semantic.logical-order","case_id":"apache-binary","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d6fd29de2111abe686c631fd5ffa090530a21d247aa75cbd35ec237ae626e33b","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary-truncated-min-max","column_order_count":6,"created_by":"parquet-rs version 55.1.0","created_by_present":true,"file":"data/binary_truncated_min_max.parquet","footer_length":1358,"leaf_count":6,"record":"file","row_group_count":1,"schema_version":2,"sha256":"94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201","size":3070} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":false,"is_min_value_exact":false,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b66","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["utf8_full_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":false,"is_min_value_exact":false,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b66","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["binary_full_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":false,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"f09f9a804b6576696e204261636f6e","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["utf8_partial_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":false,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffff0102","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["binary_partial_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b65","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["utf8_no_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b65","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["binary_no_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-bson","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/bson.parquet","footer_length":280,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129","size":412} +{"case_id":"apache-bson","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/bson.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"BSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"BSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0f000000106100010000000a620000","min_value_hex":"0c0000001061000100000000","nan_count":null,"null_count":"1","num_values":"3","path":["bson_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"3aa6c979693f790a3bdd3ce619a54d77606ca7713b1e6f34798941867bf7acee","capability_id":"semantic.logical-order","case_id":"apache-bson","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"3aa6c979693f790a3bdd3ce619a54d77606ca7713b1e6f34798941867bf7acee","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-byte-array-decimal","column_order_count":null,"created_by":"HVR 5.3.0/9 (linux_glibc2.5-x64-64bit)","created_by_present":true,"file":"data/byte_array_decimal.parquet","footer_length":119,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d","size":324} +{"case_id":"apache-byte-array-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/byte_array_decimal.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"BYTE_ARRAY","precision":4,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"360440c81d6582af1ea67d57d83160955e17954f33b22990e91c9925e58a4010","capability_id":"semantic.logical-order","case_id":"apache-byte-array-decimal","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"360440c81d6582af1ea67d57d83160955e17954f33b22990e91c9925e58a4010","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-byte-array","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build d057b39d93014fe40f5067ee4a33621e65c91552)","created_by_present":true,"file":"data/fixed_length_byte_array.parquet","footer_length":253,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7","size":4437} +{"case_id":"apache-fixed-length-byte-array","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/fixed_length_byte_array.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":4},"max_value_hex":"000003e8","min_value_hex":"00000001","nan_count":null,"null_count":"105","num_values":"1000","path":["flba_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"380dc8af752a14f0daf751283216aed586e2de90c8ca501497ee1590ae3c0946","capability_id":"semantic.logical-order","case_id":"apache-fixed-length-byte-array","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"380dc8af752a14f0daf751283216aed586e2de90c8ca501497ee1590ae3c0946","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2","size":677} +{"case_id":"apache-fixed-length-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"0000000000000000000960","deprecated_min_hex":"00000000000000000000c8","distinct_count":null,"file":"data/fixed_length_decimal.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":25,"scale":2,"time_unit":null,"type_length":11},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"42ef4e48789b3d08ff664fe2bbaa555aac36a3e36025468035902d8c28f644d9","capability_id":"semantic.logical-order","case_id":"apache-fixed-length-decimal","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"42ef4e48789b3d08ff664fe2bbaa555aac36a3e36025468035902d8c28f644d9","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal-legacy","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal_legacy.parquet","footer_length":336,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a","size":537} +{"case_id":"apache-fixed-length-decimal-legacy","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"000000000960","deprecated_min_hex":"0000000000c8","distinct_count":null,"file":"data/fixed_length_decimal_legacy.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":13,"scale":2,"time_unit":null,"type_length":6},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"d12c0777843ad5c70b9070d5bd7acb5175ea78c4a1beb2b1e0b7c4ef1739a13d","capability_id":"semantic.logical-order","case_id":"apache-fixed-length-decimal-legacy","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d12c0777843ad5c70b9070d5bd7acb5175ea78c4a1beb2b1e0b7c4ef1739a13d","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-nonzeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_nonzeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"d0117dd9655992b869f8207235526a7d8931e079fd68d68c88c0170faa1f11ee","size":501} +{"case_id":"apache-float16-nonzeros-and-nans","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0040","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/float16_nonzeros_and_nans.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0040","min_value_hex":"00c0","nan_count":null,"null_count":"1","num_values":"8","path":["x"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"e3a4975b63a78f7b1589a945dad708728ac4434df43664dade3c6be8343badb3","capability_id":"semantic.logical-order","case_id":"apache-float16-nonzeros-and-nans","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"e3a4975b63a78f7b1589a945dad708728ac4434df43664dade3c6be8343badb3","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a376a7c6f7e28fdb3dff4407707df1ad0e9acce0d67c786b70197500e4e2f78b","capability_id":"semantic.type-order","case_id":"apache-float16-nonzeros-and-nans","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a376a7c6f7e28fdb3dff4407707df1ad0e9acce0d67c786b70197500e4e2f78b","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-zeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_zeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"4901850e7dcd64588a49391fa1dddca514b598e6f4266239033ea73513850f47","size":489} +{"case_id":"apache-float16-zeros-and-nans","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000","deprecated_min_hex":"0080","distinct_count":null,"file":"data/float16_zeros_and_nans.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0000","min_value_hex":"0080","nan_count":null,"null_count":"1","num_values":"3","path":["x"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"210bdb17a1f4790dc194543c97d912d46885a7972e57957cb9ddb0875097a638","capability_id":"semantic.logical-order","case_id":"apache-float16-zeros-and-nans","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"210bdb17a1f4790dc194543c97d912d46885a7972e57957cb9ddb0875097a638","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"c9176c96f34544adc611dc5dbf0fa1b61d6cbf79aa21e0e6bd8c5f80cc533863","capability_id":"semantic.type-order","case_id":"apache-float16-zeros-and-nans","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c9176c96f34544adc611dc5dbf0fa1b61d6cbf79aa21e0e6bd8c5f80cc533863","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-floating-orders-nan-count","column_order_count":6,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build c5dcd8ca5bad5fde9c797b876a16b5bf3b9206c0)","created_by_present":true,"file":"data/floating_orders_nan_count.parquet","footer_length":3026,"leaf_count":6,"record":"file","row_group_count":5,"schema_version":2,"sha256":"17f7d7655a089b9504a828dffaccd72225a9a6fd2a697099b5336ab274386f0a","size":6143} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"00000000000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"00000000000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"00000000000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"00000000000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"00c0","nan_count":"0","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"00c0","nan_count":"0","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"00004040","deprecated_min_hex":"000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00004040","min_value_hex":"000000c0","nan_count":"4","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"4","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000000840","deprecated_min_hex":"00000000000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000000840","min_value_hex":"00000000000000c0","nan_count":"4","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"4","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0042","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0042","min_value_hex":"00c0","nan_count":"4","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":null,"min_value_hex":null,"nan_count":"4","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"ffffff7f","deprecated_min_hex":"ffffffff","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffffff7f","min_value_hex":"ffffffff","nan_count":"10","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"10","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"ffffffffffffff7f","deprecated_min_hex":"ffffffffffffffff","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffffffffffffff7f","min_value_hex":"ffffffffffffffff","nan_count":"10","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"10","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"ff7f","deprecated_min_hex":"ffff","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"ff7f","min_value_hex":"ffff","nan_count":"10","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":null,"min_value_hex":null,"nan_count":"10","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"00000000","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"00000000","nan_count":"0","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"00000080","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"00000080","nan_count":"0","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"0000000000000000","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"0000000000000000","nan_count":"0","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"0000000000000080","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"0000000000000080","nan_count":"0","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"0000","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"0000","nan_count":"0","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"0080","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"0080","nan_count":"0","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"00000080","deprecated_min_hex":"0000a0c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000080","min_value_hex":"0000a0c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"00000000","deprecated_min_hex":"0000a0c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000000","min_value_hex":"0000a0c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000000080","deprecated_min_hex":"00000000000014c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000000080","min_value_hex":"00000000000014c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000000000000000","deprecated_min_hex":"00000000000014c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000000000","min_value_hex":"00000000000014c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0080","deprecated_min_hex":"00c5","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0080","min_value_hex":"00c5","nan_count":"0","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000","deprecated_min_hex":"00c5","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0000","min_value_hex":"00c5","nan_count":"0","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"afb002da1691bee4e456fe58d88f86d884f99b9b5117c2de27279f4296b749ea","capability_id":"semantic.count-state","case_id":"apache-floating-orders-nan-count","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"afb002da1691bee4e456fe58d88f86d884f99b9b5117c2de27279f4296b749ea","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"96f1afa2bd4e2858eca7e2ac836efabb46a043e14033b48ace058e696ec62046","capability_id":"semantic.ieee-total-order","case_id":"apache-floating-orders-nan-count","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"96f1afa2bd4e2858eca7e2ac836efabb46a043e14033b48ace058e696ec62046","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f038dbf0106f59bfe1fc6ed9560568d9e90612b56e56a3b7168810c769c89edd","capability_id":"semantic.type-order","case_id":"apache-floating-orders-nan-count","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f038dbf0106f59bfe1fc6ed9560568d9e90612b56e56a3b7168810c769c89edd","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int32_decimal.parquet","footer_length":329,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9","size":478} +{"case_id":"apache-int32-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"60090000","deprecated_min_hex":"64000000","distinct_count":null,"file":"data/int32_decimal.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT32","precision":4,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"845199e959df727fae669f08c918a5aa3cbceebbb4abb303684ee485e93e446a","capability_id":"semantic.logical-order","case_id":"apache-int32-decimal","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"845199e959df727fae669f08c918a5aa3cbceebbb4abb303684ee485e93e446a","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-with-null-pages","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build 433de8df33fcf31927f7b51456be9f53e64d48b9)","created_by_present":true,"file":"data/int32_with_null_pages.parquet","footer_length":265,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7","size":3829} +{"case_id":"apache-int32-with-null-pages","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0720e57f","deprecated_min_hex":"c664a180","distinct_count":null,"file":"data/int32_with_null_pages.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0720e57f","min_value_hex":"c664a180","nan_count":null,"null_count":"275","num_values":"1000","path":["int32_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"c6034cfef502044379eac8856ec4f6a9ab1d7ede43d930893df642c3a233361f","capability_id":"semantic.count-state","case_id":"apache-int32-with-null-pages","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c6034cfef502044379eac8856ec4f6a9ab1d7ede43d930893df642c3a233361f","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"dbda6da58c536bfd6099d50962c2f26d502823a8d533a6e259008692c2ad272b","capability_id":"semantic.type-order","case_id":"apache-int32-with-null-pages","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"dbda6da58c536bfd6099d50962c2f26d502823a8d533a6e259008692c2ad272b","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int64-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int64_decimal.parquet","footer_length":338,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56","size":591} +{"case_id":"apache-int64-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"6009000000000000","deprecated_min_hex":"6400000000000000","distinct_count":null,"file":"data/int64_decimal.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT64","precision":10,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"6a0ebc4a033f53e0cb45100f84b1fcfc011e517e3de7ee852357f1a29132d1ea","capability_id":"semantic.logical-order","case_id":"apache-int64-decimal","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"6a0ebc4a033f53e0cb45100f84b1fcfc011e517e3de7ee852357f1a29132d1ea","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int96-timestamp-order","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 8931c1c55f1fba399dd75139f75bcde0b84137c0)","created_by_present":true,"file":"data/int96_timestamp_order.parquet","footer_length":271,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e35f8748d286a729e719a01c5411f81c79802d61a55046d5cf9a663918a14644","size":427} +{"case_id":"apache-int96-timestamp-order","column_order":{"field_id":3,"header_hex":"3c","state":"UNKNOWN","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/int96_timestamp_order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT96","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000000000000008d3d2500","min_value_hex":"7b00000000000000403b2500","nan_count":null,"null_count":"0","num_values":"4","path":["ts"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-json","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/json.parquet","footer_length":270,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5","size":402} +{"case_id":"apache-json","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/json.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2261223a317d","min_value_hex":"5b312c6e756c6c2c335d","nan_count":null,"null_count":"1","num_values":"4","path":["json_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"efb4c43e6675c7624ec05a80041f7b9298f2b397976b1bb39aa3666859b8ec8d","capability_id":"semantic.logical-order","case_id":"apache-json","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"efb4c43e6675c7624ec05a80041f7b9298f2b397976b1bb39aa3666859b8ec8d","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-nan-in-stats","column_order_count":1,"created_by":"parquet-cpp version 1.3.2-SNAPSHOT","created_by_present":true,"file":"data/nan_in_stats.parquet","footer_length":156,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337","size":329} +{"case_id":"apache-nan-in-stats","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"000000000000f87f","deprecated_min_hex":"000000000000f03f","distinct_count":null,"file":"data/nan_in_stats.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"000000000000f87f","min_value_hex":"000000000000f03f","nan_count":null,"null_count":"0","num_values":"2","path":["x"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"ac0aceba4c1da523d7f6a1f2084f2a28289a444d2ac4691310eb1846a597a494","capability_id":"semantic.type-order","case_id":"apache-nan-in-stats","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ac0aceba4c1da523d7f6a1f2084f2a28289a444d2ac4691310eb1846a597a494","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-rle-boolean-encoding","column_order_count":null,"created_by":null,"created_by_present":false,"file":"data/rle_boolean_encoding.parquet","footer_length":111,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76","size":192} +{"case_id":"apache-rle-boolean-encoding","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"01","deprecated_min_hex":"00","distinct_count":null,"file":"data/rle_boolean_encoding.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"6","num_values":"68","path":["datatype_boolean"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"a5ea979ba9e85c8d72ac7cfef1537bde99df7cf30d32d0495b6df65fbddd8501","capability_id":"semantic.type-order","case_id":"apache-rle-boolean-encoding","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a5ea979ba9e85c8d72ac7cfef1537bde99df7cf30d32d0495b6df65fbddd8501","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-single-nan","column_order_count":1,"created_by":"parquet-cpp version 1.5.1-SNAPSHOT","created_by_present":true,"file":"data/single_nan.parquet","footer_length":567,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"ea3371c44ed1794843a2f529888120537f68aedcb80d6fbe32cea1003ab5769e","size":660} +{"case_id":"apache-single-nan","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/single_nan.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"1","num_values":"1","path":["mycol"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"5d0ff3272b9c62d1ca308f59d9a34531a8c75db1b4bdb5b73202121d50ec5d96","capability_id":"semantic.count-state","case_id":"apache-single-nan","detail":"Frozen independent model interpreted every normalized raw column in this case.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"5d0ff3272b9c62d1ca308f59d9a34531a8c75db1b4bdb5b73202121d50ec5d96","record":"case_result","schema_version":2,"status":"PASS"} diff --git a/test/conformance/n6/evidence/parquet-java.normalized.jsonl b/test/conformance/n6/evidence/parquet-java.normalized.jsonl new file mode 100644 index 0000000..cc46a0b --- /dev/null +++ b/test/conformance/n6/evidence/parquet-java.normalized.jsonl @@ -0,0 +1,49 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-parquet-java","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"parquet-java","producer_version":"1.17.1","record":"run","schema_version":2,"source_revision":"78a8d3230eb4769db93de5f2f2e18363c04cae81","toolchain_sha256":"a4459b555778b1979de10510fd5291372ffb9f41166539868ce3639494d5a452","unsupported_cases":["apache-floating-orders-nan-count"],"upstream_evidence":[{"evidence_id":"normalized-raw-java-apache-corpus","file":"test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl","sha256":"1692d30284b57581993d524d41baa16b43e94b26dd3832bd6d63d689a1bb4ff8"}]} +{"case_id":"apache-alltypes-dictionary","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_dictionary.parquet","footer_length":723,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb","size":1698} +{"actual_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","capability_id":"read.logical-values","case_id":"apache-alltypes-dictionary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-alltypes-plain","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_plain.parquet","footer_length":730,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4","size":1851} +{"actual_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","capability_id":"read.logical-values","case_id":"apache-alltypes-plain","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build 031a6654009e3b82020012a18434c582bd74c73a)","created_by_present":true,"file":"data/binary.parquet","footer_length":371,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b","size":478} +{"actual_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","capability_id":"read.logical-values","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2","capability_id":"wire.column-order.type","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary-truncated-min-max","column_order_count":6,"created_by":"parquet-rs version 55.1.0","created_by_present":true,"file":"data/binary_truncated_min_max.parquet","footer_length":1358,"leaf_count":6,"record":"file","row_group_count":1,"schema_version":2,"sha256":"94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201","size":3070} +{"actual_sha256":"386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae","capability_id":"wire.column-order.type","case_id":"apache-binary-truncated-min-max","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-bson","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/bson.parquet","footer_length":280,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129","size":412} +{"actual_sha256":"eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a","capability_id":"read.logical-values","case_id":"apache-bson","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c","capability_id":"wire.column-order.type","case_id":"apache-bson","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-byte-array-decimal","column_order_count":null,"created_by":"HVR 5.3.0/9 (linux_glibc2.5-x64-64bit)","created_by_present":true,"file":"data/byte_array_decimal.parquet","footer_length":119,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d","size":324} +{"actual_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","capability_id":"read.logical-values","case_id":"apache-byte-array-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-byte-array","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build d057b39d93014fe40f5067ee4a33621e65c91552)","created_by_present":true,"file":"data/fixed_length_byte_array.parquet","footer_length":253,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7","size":4437} +{"actual_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","capability_id":"read.logical-values","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af","capability_id":"wire.column-order.type","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2","size":677} +{"actual_sha256":"ed5bdd8b71b0b625d793eb846f962ab257e36d1611070b4a33b2a916ec3f9e4d","capability_id":"compat.legacy-statistics","case_id":"apache-fixed-length-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ed5bdd8b71b0b625d793eb846f962ab257e36d1611070b4a33b2a916ec3f9e4d","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal-legacy","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal_legacy.parquet","footer_length":336,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a","size":537} +{"actual_sha256":"acd3c50a51cf263624be52f9e43beb2ce75939f6a9991a91084ef425df321ecb","capability_id":"compat.legacy-statistics","case_id":"apache-fixed-length-decimal-legacy","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"acd3c50a51cf263624be52f9e43beb2ce75939f6a9991a91084ef425df321ecb","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal-legacy","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-nonzeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_nonzeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"d0117dd9655992b869f8207235526a7d8931e079fd68d68c88c0170faa1f11ee","size":501} +{"actual_sha256":"495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b","capability_id":"wire.column-order.type","case_id":"apache-float16-nonzeros-and-nans","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-zeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_zeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"4901850e7dcd64588a49391fa1dddca514b598e6f4266239033ea73513850f47","size":489} +{"actual_sha256":"1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665","capability_id":"wire.column-order.type","case_id":"apache-float16-zeros-and-nans","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-floating-orders-nan-count","column_order_count":6,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build c5dcd8ca5bad5fde9c797b876a16b5bf3b9206c0)","created_by_present":true,"file":"data/floating_orders_nan_count.parquet","footer_length":3026,"leaf_count":6,"record":"file","row_group_count":5,"schema_version":2,"sha256":"17f7d7655a089b9504a828dffaccd72225a9a6fd2a697099b5336ab274386f0a","size":6143} +{"actual_sha256":null,"capability_id":"wire.statistics.nan-count","case_id":"apache-floating-orders-nan-count","detail":"The reviewed capability matrix marks this result unsupported.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":null,"record":"case_result","schema_version":2,"status":"UNSUPPORTED"} +{"case_id":"apache-int32-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int32_decimal.parquet","footer_length":329,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9","size":478} +{"actual_sha256":"d0ff4a81cfe9b6d19fdb0400828f2749288515bfc1e9a8b02a4b137c998608d5","capability_id":"compat.legacy-statistics","case_id":"apache-int32-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d0ff4a81cfe9b6d19fdb0400828f2749288515bfc1e9a8b02a4b137c998608d5","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","capability_id":"read.logical-values","case_id":"apache-int32-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-with-null-pages","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build 433de8df33fcf31927f7b51456be9f53e64d48b9)","created_by_present":true,"file":"data/int32_with_null_pages.parquet","footer_length":265,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7","size":3829} +{"actual_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","capability_id":"read.logical-values","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a","capability_id":"wire.column-order.type","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int64-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int64_decimal.parquet","footer_length":338,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56","size":591} +{"actual_sha256":"c7187495c163ebd92ad33036367bb9d88ffb6211e1340e42f982c3195d0d738d","capability_id":"compat.legacy-statistics","case_id":"apache-int64-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c7187495c163ebd92ad33036367bb9d88ffb6211e1340e42f982c3195d0d738d","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","capability_id":"read.logical-values","case_id":"apache-int64-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-json","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/json.parquet","footer_length":270,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5","size":402} +{"actual_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","capability_id":"read.logical-values","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04","capability_id":"wire.column-order.type","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-nan-in-stats","column_order_count":1,"created_by":"parquet-cpp version 1.3.2-SNAPSHOT","created_by_present":true,"file":"data/nan_in_stats.parquet","footer_length":156,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337","size":329} +{"actual_sha256":"3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7","capability_id":"wire.column-order.type","case_id":"apache-nan-in-stats","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-rle-boolean-encoding","column_order_count":null,"created_by":null,"created_by_present":false,"file":"data/rle_boolean_encoding.parquet","footer_length":111,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76","size":192} +{"actual_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","capability_id":"read.logical-values","case_id":"apache-rle-boolean-encoding","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-single-nan","column_order_count":1,"created_by":"parquet-cpp version 1.5.1-SNAPSHOT","created_by_present":true,"file":"data/single_nan.parquet","footer_length":567,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"ea3371c44ed1794843a2f529888120537f68aedcb80d6fbe32cea1003ab5769e","size":660} +{"actual_sha256":"6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d","capability_id":"wire.column-order.type","case_id":"apache-single-nan","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"cdaa409cc6dada3fada0752543aa6911174cd2523a38f996490e814bc1f58b1a","capability_id":"compat.legacy-statistics","case_id":"producer-parquet-251","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"cdaa409cc6dada3fada0752543aa6911174cd2523a38f996490e814bc1f58b1a","record":"case_result","schema_version":2,"status":"PASS"} diff --git a/test/conformance/n6/evidence/parquet-jl.normalized.jsonl b/test/conformance/n6/evidence/parquet-jl.normalized.jsonl new file mode 100644 index 0000000..9e4de41 --- /dev/null +++ b/test/conformance/n6/evidence/parquet-jl.normalized.jsonl @@ -0,0 +1,96 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-parquet-jl","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"parquet-jl","producer_version":"1.0.0-DEV","record":"run","schema_version":2,"source_revision":"ea75000a8b4505c73efe50476a45dfe427ed5c8f7123fbaf244390f2b26b0c80","toolchain_sha256":"4cf7759e22159ab3c6133bcecc3ba94b5857485581e586bf8dad49bc325a3a5e","unsupported_cases":[]} +{"case_id":"julia-writer-type-order","column_order_count":8,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-type-order.parquet","footer_length":1195,"leaf_count":8,"record":"file","row_group_count":2,"schema_version":2,"sha256":"333a9c3042992ba5f14c97db59c42fc98d4093bebcbab942cfcdfdedc43adf8a","size":1884} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"09000000","min_value_hex":"1cfbffff","nan_count":null,"null_count":"1","num_values":"3","path":["signed"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":32,"converted_type":"UINT_32","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":false,"logical_type":"INTEGER","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffffffff","min_value_hex":"00000000","nan_count":null,"null_count":"0","num_values":"3","path":["unsigned"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ff","min_value_hex":"00cf","nan_count":null,"null_count":"1","num_values":"3","path":["raw"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a","min_value_hex":"61","nan_count":null,"null_count":"1","num_values":"3","path":["text"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT32","precision":9,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":"00000000","min_value_hex":"6fa0feff","nan_count":null,"null_count":"1","num_values":"3","path":["decimal"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":"DATE","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DATE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000000","min_value_hex":"219cffff","nan_count":null,"null_count":"1","num_values":"3","path":["dates"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"1","num_values":"3","path":["flag"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":3},"max_value_hex":"ff0000","min_value_hex":"00ff28","nan_count":null,"null_count":"0","num_values":"3","path":["fixed"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ac050000","min_value_hex":"f9ffffff","nan_count":null,"null_count":"0","num_values":"3","path":["signed"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":32,"converted_type":"UINT_32","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":false,"logical_type":"INTEGER","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"86000000","min_value_hex":"01000000","nan_count":null,"null_count":"0","num_values":"3","path":["unsigned"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"80","min_value_hex":"","nan_count":null,"null_count":"0","num_values":"3","path":["raw"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a7a","min_value_hex":"610062","nan_count":null,"null_count":"0","num_values":"3","path":["text"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT32","precision":9,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":"9f860100","min_value_hex":"ffffffff","nan_count":null,"null_count":"0","num_values":"3","path":["decimal"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":"DATE","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DATE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7ab90000","min_value_hex":"ffffffff","nan_count":null,"null_count":"0","num_values":"3","path":["dates"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"3","path":["flag"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":3},"max_value_hex":"800000","min_value_hex":"010203","nan_count":null,"null_count":"0","num_values":"3","path":["fixed"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"e5281ff1b4efb860e8a5b166b8afe34e696a379965038d233ea194112bc3766b","capability_id":"write.type-order","case_id":"julia-writer-type-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"e5281ff1b4efb860e8a5b166b8afe34e696a379965038d233ea194112bc3766b","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"9bbba11f04d0392c30905be9525ac4f95b8a559d56592d1aeaa731aac142a62c","capability_id":"semantic.type-order","case_id":"julia-writer-type-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"9bbba11f04d0392c30905be9525ac4f95b8a559d56592d1aeaa731aac142a62c","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a1e44e16eefcefda61f79ae126a7ff5d5e054f6efa428f1d2c2a242817cebc74","capability_id":"semantic.logical-order","case_id":"julia-writer-type-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a1e44e16eefcefda61f79ae126a7ff5d5e054f6efa428f1d2c2a242817cebc74","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a337b29bbb2496f7c13ea27430609f4f96606e5a872ababe8b5b2dfceec73026","capability_id":"read.logical-values","case_id":"julia-writer-type-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a337b29bbb2496f7c13ea27430609f4f96606e5a872ababe8b5b2dfceec73026","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-ieee-order","column_order_count":3,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-ieee-order.parquet","footer_length":531,"leaf_count":3,"record":"file","row_group_count":2,"schema_version":2,"sha256":"8782589a7b6bfcb576acf04e0bc15cef156b931497f63a5623c7cb2b4355188c","size":907} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"007c","min_value_hex":"00bc","nan_count":"1","null_count":"1","num_values":"6","path":["half"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000807f","min_value_hex":"000080bf","nan_count":"1","null_count":"1","num_values":"6","path":["single"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"000000000000f07f","min_value_hex":"000000000000f0bf","nan_count":"1","null_count":"1","num_values":"6","path":["double"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"107e","min_value_hex":"20fe","nan_count":"6","null_count":"0","num_values":"6","path":["half"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"2222e27f","min_value_hex":"2143c5ff","nan_count":"6","null_count":"0","num_values":"6","path":["single"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"452301000000f87f","min_value_hex":"220000000000f8ff","nan_count":"6","null_count":"0","num_values":"6","path":["double"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"f6723f2dd8e2f2e4eaab01d50d52a3d3332c8a2b15a4c36f4ae45028cfed55a6","capability_id":"write.type-order","case_id":"julia-writer-ieee-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f6723f2dd8e2f2e4eaab01d50d52a3d3332c8a2b15a4c36f4ae45028cfed55a6","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"67184ff5b5cc13d85d46d5b515ac82c55701f276b3c4fe2e413cc79a72bbdba9","capability_id":"semantic.ieee-total-order","case_id":"julia-writer-ieee-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"67184ff5b5cc13d85d46d5b515ac82c55701f276b3c4fe2e413cc79a72bbdba9","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"4bacd2757f46ca09b08445b54859d9e896ea556ec8839ce82e6aa798c91e5d81","capability_id":"semantic.count-state","case_id":"julia-writer-ieee-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"4bacd2757f46ca09b08445b54859d9e896ea556ec8839ce82e6aa798c91e5d81","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"3145325e5adcacfc33a44cb661c17304ec7b2defb75b7135058f3b5f634279cc","capability_id":"read.logical-values","case_id":"julia-writer-ieee-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"3145325e5adcacfc33a44cb661c17304ec7b2defb75b7135058f3b5f634279cc","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-undefined-order","column_order_count":3,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-undefined-order.parquet","footer_length":274,"leaf_count":3,"record":"file","row_group_count":1,"schema_version":2,"sha256":"ce53d8ec5024f2694bc5a50782c918f1368b39ae4d8c94bc80791e048bb28648","size":454} +{"case_id":"julia-writer-undefined-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-undefined-order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"INTERVAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"INTERVAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":12},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"4","path":["first"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-undefined-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-undefined-order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":"INTERVAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"INTERVAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":12},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"2","num_values":"4","path":["second"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-undefined-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-undefined-order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"UNKNOWN","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"4","num_values":"4","path":["unknown"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"1af58a5de2a2af52276fd50bdffc32966f8afd79ee3e7192119a766e6e94ff27","capability_id":"write.type-order","case_id":"julia-writer-undefined-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"1af58a5de2a2af52276fd50bdffc32966f8afd79ee3e7192119a766e6e94ff27","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"7bbd807b775ff45acc4bed7bd27bb7b1a7e3b7ba03261a3cb5282e35c1663dab","capability_id":"semantic.type-order","case_id":"julia-writer-undefined-order","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7bbd807b775ff45acc4bed7bd27bb7b1a7e3b7ba03261a3cb5282e35c1663dab","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-statistics-disabled","column_order_count":null,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-statistics-disabled.parquet","footer_length":285,"leaf_count":2,"record":"file","row_group_count":2,"schema_version":2,"sha256":"b0be13b0030888a41c82de57fafa987b29997352cd923c7542539b34a39141fd","size":494} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["number"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["text"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["number"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["text"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"54a198c6b8fd94996b7cb84c9581b5bc148d41f98fce867456132ef7b6a7e773","capability_id":"write.statistics-disabled","case_id":"julia-writer-statistics-disabled","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"54a198c6b8fd94996b7cb84c9581b5bc148d41f98fce867456132ef7b6a7e773","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"3cfc9c5723699d1755149924c9f3199d0fd15ba0c7220facb62cc9fefe04316f","capability_id":"read.logical-values","case_id":"julia-writer-statistics-disabled","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"3cfc9c5723699d1755149924c9f3199d0fd15ba0c7220facb62cc9fefe04316f","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-oversized-bounds","column_order_count":1,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-oversized-bounds.parquet","footer_length":141,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"0dc68216f7dd5b1ef425b59e398553af71f254fe249c7dba30de82d25d491c9a","size":12494} +{"case_id":"julia-writer-oversized-bounds","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-oversized-bounds.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"3","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"e7d8042e29d78a5260e9c4eeddcdcd51421f04ba470a6e5fd2fd407689ac8c61","capability_id":"write.type-order","case_id":"julia-writer-oversized-bounds","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"e7d8042e29d78a5260e9c4eeddcdcd51421f04ba470a6e5fd2fd407689ac8c61","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"595da8ef8e837ce9604ea898e2e85157c74f6a077dbe5d1c295495abd6e0ade3","capability_id":"write.statistics-limit","case_id":"julia-writer-oversized-bounds","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"595da8ef8e837ce9604ea898e2e85157c74f6a077dbe5d1c295495abd6e0ade3","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-nested-row-groups","column_order_count":4,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-nested-row-groups.parquet","footer_length":673,"leaf_count":4,"record":"file","row_group_count":2,"schema_version":2,"sha256":"9a705db14a5c00260717bc907a916303ce297a039c03fb0f3ae981022e763d49","size":1025} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"62","min_value_hex":"61","nan_count":null,"null_count":"1","num_values":"3","path":["tag"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"a4000000","min_value_hex":"02000000","nan_count":null,"null_count":"1","num_values":"3","path":["rows","id"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"3","num_values":"3","path":["rows","items","list","element"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"3","path":["flag"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a","min_value_hex":"63","nan_count":null,"null_count":"1","num_values":"3","path":["tag"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"05000000","min_value_hex":"03000000","nan_count":null,"null_count":"0","num_values":"3","path":["rows","id"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0a000000","min_value_hex":"ffffffff","nan_count":null,"null_count":"2","num_values":"6","path":["rows","items","list","element"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"3","path":["flag"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"df7c774bba47dcddcc83492547d82277cd6bfbbadd44cfa1f4d22508410cb93f","capability_id":"write.type-order","case_id":"julia-writer-nested-row-groups","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"df7c774bba47dcddcc83492547d82277cd6bfbbadd44cfa1f4d22508410cb93f","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"e99fd1fc9846c22eae5ca1c511239dafa606a578a6a75eeb4a059627e0bbdfea","capability_id":"semantic.type-order","case_id":"julia-writer-nested-row-groups","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"e99fd1fc9846c22eae5ca1c511239dafa606a578a6a75eeb4a059627e0bbdfea","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a92e3459e9c635700bbc8e9af38b112029d99201f4a8da864318b7467f38a35c","capability_id":"semantic.count-state","case_id":"julia-writer-nested-row-groups","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a92e3459e9c635700bbc8e9af38b112029d99201f4a8da864318b7467f38a35c","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"894e9a474f79a942bb4d7ca85121992373ed2fea71f520efa993184054bfe1c5","capability_id":"read.logical-values","case_id":"julia-writer-nested-row-groups","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"894e9a474f79a942bb4d7ca85121992373ed2fea71f520efa993184054bfe1c5","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-reader-untrusted-producer","column_order_count":2,"created_by":"parquet-mr version 1.7.0 (build n6)","created_by_present":true,"file":"generated/julia-reader-untrusted-producer.parquet","footer_length":370,"leaf_count":2,"record":"file","row_group_count":2,"schema_version":2,"sha256":"57f7c7b1d350fc0f84bb1de006aa7ed992194f1a12eea28a506b97e736039b93","size":575} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a","min_value_hex":"61","nan_count":null,"null_count":"1","num_values":"3","path":["binary"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"09000000","deprecated_min_hex":"f6ffffff","distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"1","num_values":"3","path":["signed"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7461696c","min_value_hex":"6d6964646c65","nan_count":null,"null_count":"0","num_values":"3","path":["binary"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"b6000000","deprecated_min_hex":"9dffffff","distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"3","path":["signed"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"b7fa2efc15dac615aad39c2db33e2369f45aaaf0fd78008403e6444bb1399016","capability_id":"semantic.producer-trust","case_id":"julia-reader-untrusted-producer","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b7fa2efc15dac615aad39c2db33e2369f45aaaf0fd78008403e6444bb1399016","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f0146a8d6334ce2f96bafbfa445535e88abf264270cdd5a42b551049e86fd627","capability_id":"compat.legacy-statistics","case_id":"julia-reader-untrusted-producer","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f0146a8d6334ce2f96bafbfa445535e88abf264270cdd5a42b551049e86fd627","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"ed119709365e0bcb426792b57a4cb50f8c347717ee1e1e46a5278c6c3a160318","capability_id":"read.logical-values","case_id":"julia-reader-untrusted-producer","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ed119709365e0bcb426792b57a4cb50f8c347717ee1e1e46a5278c6c3a160318","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-reader-no-pruning-absent","column_order_count":null,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-reader-no-pruning-absent.parquet","footer_length":177,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"a817b4c27f01744c853aa09399c002127463940eb9a94f5b31b7fa57b33e2927","size":321} +{"case_id":"julia-reader-no-pruning-absent","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-absent.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-absent","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-absent.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.no-pruning","case_id":"julia-reader-no-pruning-absent","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.logical-values","case_id":"julia-reader-no-pruning-absent","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-reader-no-pruning-trusted","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build n6)","created_by_present":true,"file":"generated/julia-reader-no-pruning-trusted.parquet","footer_length":250,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"b9e499b621048f4e72d25cef732e800bfe380ee03d0eedce46de67602bf2767f","size":394} +{"case_id":"julia-reader-no-pruning-trusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-trusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2276616c7565223a32337d","min_value_hex":"5b312c322c335d","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-trusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-trusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b226e6573746564223a747275657d","min_value_hex":"227461696c22","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.no-pruning","case_id":"julia-reader-no-pruning-trusted","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.logical-values","case_id":"julia-reader-no-pruning-trusted","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-reader-no-pruning-untrusted","column_order_count":1,"created_by":"parquet-mr version 1.7.0 (build n6)","created_by_present":true,"file":"generated/julia-reader-no-pruning-untrusted.parquet","footer_length":249,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"22b92a7f1139a6d7ce8f33a5440b4891db9eae323c5e3d0c620969d3f5761771","size":393} +{"case_id":"julia-reader-no-pruning-untrusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-untrusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2276616c7565223a32337d","min_value_hex":"5b312c322c335d","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-untrusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-untrusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b226e6573746564223a747275657d","min_value_hex":"227461696c22","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.no-pruning","case_id":"julia-reader-no-pruning-untrusted","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.logical-values","case_id":"julia-reader-no-pruning-untrusted","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-reader-no-pruning-oversized","column_order_count":1,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-reader-no-pruning-oversized.parquet","footer_length":16594,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"1e416cfdbb7b25c5a7e0013845a24ef2ff28e2948b326bacb9eeb38b8e7862a9","size":16738} +{"case_id":"julia-reader-no-pruning-oversized","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-oversized.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","min_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-oversized","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-oversized.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","min_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.no-pruning","case_id":"julia-reader-no-pruning-oversized","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.logical-values","case_id":"julia-reader-no-pruning-oversized","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-reader-no-pruning-invalid","column_order_count":1,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-reader-no-pruning-invalid.parquet","footer_length":235,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"360c5e75bd2fa90a28e05ba46990f44f98b5e32452555d7c7f84ce5db7454d20","size":379} +{"case_id":"julia-reader-no-pruning-invalid","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-invalid.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2276616c7565223a32337d","min_value_hex":"000000","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-invalid","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-invalid.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b226e6573746564223a747275657d","min_value_hex":"000000","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.no-pruning","case_id":"julia-reader-no-pruning-invalid","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","capability_id":"read.logical-values","case_id":"julia-reader-no-pruning-invalid","detail":"Parquet.jl N6 harness assertion passed","digest_contract":"n6-no-pruning-trace-sha256-v1","expected_sha256":"fc1f904014ecd262c643ec1111f0d54ba3ce1d986975ff5db5a43ae15b7f2e06","record":"case_result","schema_version":2,"status":"PASS"} diff --git a/test/conformance/n6/evidence/pyarrow.normalized.jsonl b/test/conformance/n6/evidence/pyarrow.normalized.jsonl new file mode 100644 index 0000000..9155fa9 --- /dev/null +++ b/test/conformance/n6/evidence/pyarrow.normalized.jsonl @@ -0,0 +1,38 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-pyarrow","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"pyarrow","producer_version":"25.0.1","record":"run","schema_version":2,"source_revision":"beccec0d0c451b7aa3e4530416ac431b3c035c69","toolchain_sha256":"0e1e7fa951f82f12ddfdbeb3865936763c1818f1f20a655d2646b1186cbcd958","unsupported_cases":["apache-floating-orders-nan-count"],"upstream_evidence":[{"evidence_id":"normalized-raw-java-apache-corpus","file":"test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl","sha256":"1692d30284b57581993d524d41baa16b43e94b26dd3832bd6d63d689a1bb4ff8"}]} +{"case_id":"apache-alltypes-dictionary","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_dictionary.parquet","footer_length":723,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb","size":1698} +{"actual_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","capability_id":"read.logical-values","case_id":"apache-alltypes-dictionary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-alltypes-plain","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_plain.parquet","footer_length":730,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4","size":1851} +{"actual_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","capability_id":"read.logical-values","case_id":"apache-alltypes-plain","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build 031a6654009e3b82020012a18434c582bd74c73a)","created_by_present":true,"file":"data/binary.parquet","footer_length":371,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b","size":478} +{"actual_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","capability_id":"read.logical-values","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"86bd6dafb44a761e68dd4dd305ceac02c2450bf8979a324bce0df76632ffe2e8","capability_id":"read.statistics-metadata","case_id":"apache-binary","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"86bd6dafb44a761e68dd4dd305ceac02c2450bf8979a324bce0df76632ffe2e8","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary-truncated-min-max","column_order_count":6,"created_by":"parquet-rs version 55.1.0","created_by_present":true,"file":"data/binary_truncated_min_max.parquet","footer_length":1358,"leaf_count":6,"record":"file","row_group_count":1,"schema_version":2,"sha256":"94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201","size":3070} +{"actual_sha256":"842e52952cef2637ea7e2476a538bd54c32371eb4f561ff56fd5e7d3d150a516","capability_id":"read.statistics-metadata","case_id":"apache-binary-truncated-min-max","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"842e52952cef2637ea7e2476a538bd54c32371eb4f561ff56fd5e7d3d150a516","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-bson","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/bson.parquet","footer_length":280,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129","size":412} +{"actual_sha256":"eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a","capability_id":"read.logical-values","case_id":"apache-bson","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"daabeb36eb56a87fe22c5e6a076095d6abb6a3cce9422d86c00b83c5d1e81225","capability_id":"read.statistics-metadata","case_id":"apache-bson","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"daabeb36eb56a87fe22c5e6a076095d6abb6a3cce9422d86c00b83c5d1e81225","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-byte-array-decimal","column_order_count":null,"created_by":"HVR 5.3.0/9 (linux_glibc2.5-x64-64bit)","created_by_present":true,"file":"data/byte_array_decimal.parquet","footer_length":119,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d","size":324} +{"actual_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","capability_id":"read.logical-values","case_id":"apache-byte-array-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-byte-array","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build d057b39d93014fe40f5067ee4a33621e65c91552)","created_by_present":true,"file":"data/fixed_length_byte_array.parquet","footer_length":253,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7","size":4437} +{"actual_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","capability_id":"read.logical-values","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"cec5d03c3c6d4fbace4831bb715cdacb02daf4783859ced5b82f0316b3af3403","capability_id":"read.statistics-metadata","case_id":"apache-fixed-length-byte-array","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"cec5d03c3c6d4fbace4831bb715cdacb02daf4783859ced5b82f0316b3af3403","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2","size":677} +{"actual_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal-legacy","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal_legacy.parquet","footer_length":336,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a","size":537} +{"actual_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","capability_id":"read.logical-values","case_id":"apache-fixed-length-decimal-legacy","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-floating-orders-nan-count","column_order_count":6,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build c5dcd8ca5bad5fde9c797b876a16b5bf3b9206c0)","created_by_present":true,"file":"data/floating_orders_nan_count.parquet","footer_length":3026,"leaf_count":6,"record":"file","row_group_count":5,"schema_version":2,"sha256":"17f7d7655a089b9504a828dffaccd72225a9a6fd2a697099b5336ab274386f0a","size":6143} +{"actual_sha256":null,"capability_id":"wire.statistics.nan-count","case_id":"apache-floating-orders-nan-count","detail":"The reviewed capability matrix marks this result unsupported.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":null,"record":"case_result","schema_version":2,"status":"UNSUPPORTED"} +{"case_id":"apache-int32-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int32_decimal.parquet","footer_length":329,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9","size":478} +{"actual_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","capability_id":"read.logical-values","case_id":"apache-int32-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-with-null-pages","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build 433de8df33fcf31927f7b51456be9f53e64d48b9)","created_by_present":true,"file":"data/int32_with_null_pages.parquet","footer_length":265,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7","size":3829} +{"actual_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","capability_id":"read.logical-values","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"c1a177f64c606180c899c0a811b5e65a9ffdee0e67c6525059c3eac2c447efdf","capability_id":"read.statistics-metadata","case_id":"apache-int32-with-null-pages","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c1a177f64c606180c899c0a811b5e65a9ffdee0e67c6525059c3eac2c447efdf","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int64-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int64_decimal.parquet","footer_length":338,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56","size":591} +{"actual_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","capability_id":"read.logical-values","case_id":"apache-int64-decimal","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-json","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/json.parquet","footer_length":270,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5","size":402} +{"actual_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","capability_id":"read.logical-values","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"d177b1ef6af0c1e78463e7578527fcedee5ea1f552dd004a855faa865ee6586d","capability_id":"read.statistics-metadata","case_id":"apache-json","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d177b1ef6af0c1e78463e7578527fcedee5ea1f552dd004a855faa865ee6586d","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-nan-in-stats","column_order_count":1,"created_by":"parquet-cpp version 1.3.2-SNAPSHOT","created_by_present":true,"file":"data/nan_in_stats.parquet","footer_length":156,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337","size":329} +{"actual_sha256":"a014089dbabbb3a1048655d74fb0dd6f5c9e23c69780e733cf21f0fe33d85291","capability_id":"read.statistics-metadata","case_id":"apache-nan-in-stats","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a014089dbabbb3a1048655d74fb0dd6f5c9e23c69780e733cf21f0fe33d85291","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-rle-boolean-encoding","column_order_count":null,"created_by":null,"created_by_present":false,"file":"data/rle_boolean_encoding.parquet","footer_length":111,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76","size":192} +{"actual_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","capability_id":"read.logical-values","case_id":"apache-rle-boolean-encoding","detail":"Observed values match the frozen canonical digest.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa","record":"case_result","schema_version":2,"status":"PASS"} diff --git a/test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl b/test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl new file mode 100644 index 0000000..4d29ffc --- /dev/null +++ b/test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl @@ -0,0 +1,150 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-raw-java-apache-corpus","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"n6-raw-java","producer_version":"parquet-2.13-raw-footer-v3","record":"run","schema_version":2,"source_revision":"c47e2a66e88943fc46fde1b028a9432f14fdf5c0","toolchain_sha256":"8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629","unsupported_cases":["atomic-bound-family"]} +{"case_id":"apache-alltypes-dictionary","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_dictionary.parquet","footer_length":723,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb","size":1698} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["id"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["bool_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["tinyint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["smallint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["int_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT64","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["bigint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["float_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["double_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":8,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["date_string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":9,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-dictionary","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_dictionary.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":10,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT96","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"2","path":["timestamp_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order_count":null,"created_by":"impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)","created_by_present":true,"file":"data/alltypes_plain.parquet","footer_length":730,"leaf_count":11,"record":"file","row_group_count":1,"schema_version":2,"sha256":"12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4","size":1851} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["id"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["bool_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["tinyint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["smallint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["int_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT64","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["bigint_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["float_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["double_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":8,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["date_string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":9,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["string_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-alltypes-plain","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/alltypes_plain.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":10,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT96","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"8","path":["timestamp_col"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build 031a6654009e3b82020012a18434c582bd74c73a)","created_by_present":true,"file":"data/binary.parquet","footer_length":371,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b","size":478} +{"case_id":"apache-binary","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0b","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"12","path":["foo"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2","capability_id":"wire.column-order.type","case_id":"apache-binary","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"9d984978b91217281c29a9642a731cd8a3b9363d86dd54a4f95365f9fd9593e9","capability_id":"wire.statistics.counts","case_id":"apache-binary","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"9d984978b91217281c29a9642a731cd8a3b9363d86dd54a4f95365f9fd9593e9","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a3197556df221a4667c947c4d0adcd33e13b2e543c90991bc5ee87126ced77ed","capability_id":"wire.statistics.modern-bounds","case_id":"apache-binary","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a3197556df221a4667c947c4d0adcd33e13b2e543c90991bc5ee87126ced77ed","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-binary-truncated-min-max","column_order_count":6,"created_by":"parquet-rs version 55.1.0","created_by_present":true,"file":"data/binary_truncated_min_max.parquet","footer_length":1358,"leaf_count":6,"record":"file","row_group_count":1,"schema_version":2,"sha256":"94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201","size":3070} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":false,"is_min_value_exact":false,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b66","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["utf8_full_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":false,"is_min_value_exact":false,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b66","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["binary_full_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":false,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"f09f9a804b6576696e204261636f6e","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["utf8_partial_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":false,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffff0102","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["binary_partial_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b65","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["utf8_no_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-binary-truncated-min-max","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/binary_truncated_min_max.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"4b65","min_value_hex":"416c","nan_count":null,"null_count":"0","num_values":"12","path":["binary_no_truncation"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae","capability_id":"wire.column-order.type","case_id":"apache-binary-truncated-min-max","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"86446fbb7f2e35869f76318a823dccd8cf495a7b0c2f2e97031117728e63d0ed","capability_id":"wire.statistics.counts","case_id":"apache-binary-truncated-min-max","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"86446fbb7f2e35869f76318a823dccd8cf495a7b0c2f2e97031117728e63d0ed","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"73b1db4b44523cb4b146b7a3ab7aa88e8a049550b8e2d175b34c76d8aecbd3bc","capability_id":"wire.statistics.exactness","case_id":"apache-binary-truncated-min-max","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"73b1db4b44523cb4b146b7a3ab7aa88e8a049550b8e2d175b34c76d8aecbd3bc","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a6775ea00b720a62d24e4867df4fc65899bbfb8dbf42658378d058f9133f68fd","capability_id":"wire.statistics.modern-bounds","case_id":"apache-binary-truncated-min-max","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a6775ea00b720a62d24e4867df4fc65899bbfb8dbf42658378d058f9133f68fd","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-bson","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/bson.parquet","footer_length":280,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129","size":412} +{"case_id":"apache-bson","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/bson.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"BSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"BSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0f000000106100010000000a620000","min_value_hex":"0c0000001061000100000000","nan_count":null,"null_count":"1","num_values":"3","path":["bson_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c","capability_id":"wire.column-order.type","case_id":"apache-bson","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"cfe16939325657fe4a4198e36cbbeb6a7129a8d920680ac3838e20479c4bc42f","capability_id":"wire.statistics.counts","case_id":"apache-bson","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"cfe16939325657fe4a4198e36cbbeb6a7129a8d920680ac3838e20479c4bc42f","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"c5b4c0e03ae7dc5de583a2fe35e61971140f05eedd2543e75e86caee25cd89c6","capability_id":"wire.statistics.modern-bounds","case_id":"apache-bson","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c5b4c0e03ae7dc5de583a2fe35e61971140f05eedd2543e75e86caee25cd89c6","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-byte-array-decimal","column_order_count":null,"created_by":"HVR 5.3.0/9 (linux_glibc2.5-x64-64bit)","created_by_present":true,"file":"data/byte_array_decimal.parquet","footer_length":119,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d","size":324} +{"case_id":"apache-byte-array-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/byte_array_decimal.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"BYTE_ARRAY","precision":4,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-fixed-length-byte-array","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build d057b39d93014fe40f5067ee4a33621e65c91552)","created_by_present":true,"file":"data/fixed_length_byte_array.parquet","footer_length":253,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7","size":4437} +{"case_id":"apache-fixed-length-byte-array","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/fixed_length_byte_array.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":4},"max_value_hex":"000003e8","min_value_hex":"00000001","nan_count":null,"null_count":"105","num_values":"1000","path":["flba_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af","capability_id":"wire.column-order.type","case_id":"apache-fixed-length-byte-array","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a9323a46864972c2da9cd0f337fc3bac549b0bd644bfe4c45db2a2df2bfb75da","capability_id":"wire.statistics.counts","case_id":"apache-fixed-length-byte-array","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a9323a46864972c2da9cd0f337fc3bac549b0bd644bfe4c45db2a2df2bfb75da","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"b531d538dc08c04ad38cb6929134518adce65546ed67395326ac4691117effd2","capability_id":"wire.statistics.modern-bounds","case_id":"apache-fixed-length-byte-array","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b531d538dc08c04ad38cb6929134518adce65546ed67395326ac4691117effd2","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2","size":677} +{"case_id":"apache-fixed-length-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"0000000000000000000960","deprecated_min_hex":"00000000000000000000c8","distinct_count":null,"file":"data/fixed_length_decimal.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":25,"scale":2,"time_unit":null,"type_length":11},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"00689badb9181aba11f16b80435c1e8fb7e3111918b5ac52e31ef9ce54b50aaa","capability_id":"wire.statistics.counts","case_id":"apache-fixed-length-decimal","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"00689badb9181aba11f16b80435c1e8fb7e3111918b5ac52e31ef9ce54b50aaa","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"c8b4c7066a814bfbb72512ac6066a8fe2420c2525629fd6ea1eec6dc020fb3c5","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-fixed-length-decimal","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c8b4c7066a814bfbb72512ac6066a8fe2420c2525629fd6ea1eec6dc020fb3c5","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-fixed-length-decimal-legacy","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/fixed_length_decimal_legacy.parquet","footer_length":336,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a","size":537} +{"case_id":"apache-fixed-length-decimal-legacy","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"000000000960","deprecated_min_hex":"0000000000c8","distinct_count":null,"file":"data/fixed_length_decimal_legacy.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":13,"scale":2,"time_unit":null,"type_length":6},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"0991d96a88cdbd106d5aaed13a2103ec592191aa984a80dd493a1564320ce93e","capability_id":"wire.statistics.counts","case_id":"apache-fixed-length-decimal-legacy","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"0991d96a88cdbd106d5aaed13a2103ec592191aa984a80dd493a1564320ce93e","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"4086e4b49afedeb1739169a17f1959931e1007ac5e12a12bb4c8e315fc88f383","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-fixed-length-decimal-legacy","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"4086e4b49afedeb1739169a17f1959931e1007ac5e12a12bb4c8e315fc88f383","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-nonzeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_nonzeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"d0117dd9655992b869f8207235526a7d8931e079fd68d68c88c0170faa1f11ee","size":501} +{"case_id":"apache-float16-nonzeros-and-nans","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0040","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/float16_nonzeros_and_nans.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0040","min_value_hex":"00c0","nan_count":null,"null_count":"1","num_values":"8","path":["x"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b","capability_id":"wire.column-order.type","case_id":"apache-float16-nonzeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"770d644959567ce166a426462e0f32e0b3a51cd361fb3d3340688b2177dcf647","capability_id":"wire.statistics.counts","case_id":"apache-float16-nonzeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"770d644959567ce166a426462e0f32e0b3a51cd361fb3d3340688b2177dcf647","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"d0f63c06d8e65ac7bce56d9d60d916dd4e7070970d9f13f8d6d77a67880590ab","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-float16-nonzeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d0f63c06d8e65ac7bce56d9d60d916dd4e7070970d9f13f8d6d77a67880590ab","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"5fa83d6109fe39c1ea95bb2e33bc04acad9d5c54d03abfcb834613f30edd5164","capability_id":"wire.statistics.modern-bounds","case_id":"apache-float16-nonzeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"5fa83d6109fe39c1ea95bb2e33bc04acad9d5c54d03abfcb834613f30edd5164","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-float16-zeros-and-nans","column_order_count":1,"created_by":"parquet-cpp-arrow version 15.0.0-SNAPSHOT","created_by_present":true,"file":"data/float16_zeros_and_nans.parquet","footer_length":346,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"4901850e7dcd64588a49391fa1dddca514b598e6f4266239033ea73513850f47","size":489} +{"case_id":"apache-float16-zeros-and-nans","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000","deprecated_min_hex":"0080","distinct_count":null,"file":"data/float16_zeros_and_nans.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0000","min_value_hex":"0080","nan_count":null,"null_count":"1","num_values":"3","path":["x"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665","capability_id":"wire.column-order.type","case_id":"apache-float16-zeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"976f0ff936783c93aa6517813d48301f7efd9b9734ffcf365a8152b5f1943902","capability_id":"wire.statistics.counts","case_id":"apache-float16-zeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"976f0ff936783c93aa6517813d48301f7efd9b9734ffcf365a8152b5f1943902","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"2364ee3ccf50dbff0ea469b4d970bda214b9963fe17c493846b99feceffa0c24","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-float16-zeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"2364ee3ccf50dbff0ea469b4d970bda214b9963fe17c493846b99feceffa0c24","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"56da23388b6b67c66487b121a8c5030e136ffb60cacac15e9834d8b7132da1fc","capability_id":"wire.statistics.modern-bounds","case_id":"apache-float16-zeros-and-nans","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"56da23388b6b67c66487b121a8c5030e136ffb60cacac15e9834d8b7132da1fc","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-floating-orders-nan-count","column_order_count":6,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build c5dcd8ca5bad5fde9c797b876a16b5bf3b9206c0)","created_by_present":true,"file":"data/floating_orders_nan_count.parquet","footer_length":3026,"leaf_count":6,"record":"file","row_group_count":5,"schema_version":2,"sha256":"17f7d7655a089b9504a828dffaccd72225a9a6fd2a697099b5336ab274386f0a","size":6143} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"00000000000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"00000000000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"00000000000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"00000000000000c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"00c0","nan_count":"0","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"00c0","nan_count":"0","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"00004040","deprecated_min_hex":"000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00004040","min_value_hex":"000000c0","nan_count":"4","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"4","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000000840","deprecated_min_hex":"00000000000000c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000000840","min_value_hex":"00000000000000c0","nan_count":"4","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"4","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0042","deprecated_min_hex":"00c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0042","min_value_hex":"00c0","nan_count":"4","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":null,"min_value_hex":null,"nan_count":"4","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"ffffff7f","deprecated_min_hex":"ffffffff","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffffff7f","min_value_hex":"ffffffff","nan_count":"10","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"10","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"ffffffffffffff7f","deprecated_min_hex":"ffffffffffffffff","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffffffffffffff7f","min_value_hex":"ffffffffffffffff","nan_count":"10","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":"10","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"ff7f","deprecated_min_hex":"ffff","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"ff7f","min_value_hex":"ffff","nan_count":"10","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":null,"min_value_hex":null,"nan_count":"10","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":2,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"00000000","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"00000000","nan_count":"0","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000a040","deprecated_min_hex":"00000080","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000a040","min_value_hex":"00000080","nan_count":"0","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"0000000000000000","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"0000000000000000","nan_count":"0","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000000000001440","deprecated_min_hex":"0000000000000080","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000001440","min_value_hex":"0000000000000080","nan_count":"0","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"0000","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"0000","nan_count":"0","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0045","deprecated_min_hex":"0080","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0045","min_value_hex":"0080","nan_count":"0","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":3,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"00000080","deprecated_min_hex":"0000a0c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000080","min_value_hex":"0000a0c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_ieee754"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"00000000","deprecated_min_hex":"0000a0c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000000","min_value_hex":"0000a0c0","nan_count":"0","null_count":"0","num_values":"10","path":["float_typedef"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0000000000000080","deprecated_min_hex":"00000000000014c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000000080","min_value_hex":"00000000000014c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_ieee754"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000000000000000","deprecated_min_hex":"00000000000014c0","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000000000000000","min_value_hex":"00000000000014c0","nan_count":"0","null_count":"0","num_values":"10","path":["double_typedef"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":"0080","deprecated_min_hex":"00c5","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0080","min_value_hex":"00c5","nan_count":"0","null_count":"0","num_values":"10","path":["float16_ieee754"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"apache-floating-orders-nan-count","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0000","deprecated_min_hex":"00c5","distinct_count":null,"file":"data/floating_orders_nan_count.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"0000","min_value_hex":"00c5","nan_count":"0","null_count":"0","num_values":"10","path":["float16_typedef"],"record":"column_statistics","row_group":4,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"2b8db92a5abe75b831808bf514973530429d5d1dba64752e3c3dbe7fef4cf244","capability_id":"wire.column-order.ieee","case_id":"apache-floating-orders-nan-count","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"2b8db92a5abe75b831808bf514973530429d5d1dba64752e3c3dbe7fef4cf244","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"623e87e2683151175afd0c3cee27bd2aa30a3215334497e66325f7b462ec7f8b","capability_id":"wire.column-order.type","case_id":"apache-floating-orders-nan-count","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"623e87e2683151175afd0c3cee27bd2aa30a3215334497e66325f7b462ec7f8b","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"9362a103defa735fc9f1b7e1ca249337f5d1c5361a49e2f416d51cab65267944","capability_id":"wire.statistics.counts","case_id":"apache-floating-orders-nan-count","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"9362a103defa735fc9f1b7e1ca249337f5d1c5361a49e2f416d51cab65267944","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"91bb1e9f06b3d088b78340cb5e604868d0346cb3bbd155976abf3734989b4dc8","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-floating-orders-nan-count","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"91bb1e9f06b3d088b78340cb5e604868d0346cb3bbd155976abf3734989b4dc8","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"16d64c02b911bf0b1adc4cef538c58ffad54b689eb3f8e207b045fda09f52127","capability_id":"wire.statistics.modern-bounds","case_id":"apache-floating-orders-nan-count","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"16d64c02b911bf0b1adc4cef538c58ffad54b689eb3f8e207b045fda09f52127","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"16da2f1eab53782892b122a42e6134692a23ad2ec43751b9885886ce1af79783","capability_id":"wire.statistics.nan-count","case_id":"apache-floating-orders-nan-count","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"16da2f1eab53782892b122a42e6134692a23ad2ec43751b9885886ce1af79783","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int32_decimal.parquet","footer_length":329,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9","size":478} +{"case_id":"apache-int32-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"60090000","deprecated_min_hex":"64000000","distinct_count":null,"file":"data/int32_decimal.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT32","precision":4,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"93c172caa5ca0fa156f12952e1aa3355148017a7d89a4799c7ee2ea35e54222b","capability_id":"wire.statistics.counts","case_id":"apache-int32-decimal","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"93c172caa5ca0fa156f12952e1aa3355148017a7d89a4799c7ee2ea35e54222b","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"24b740aac5dd539de0a5f16d4b965e6c8aeaf68f8907afaa2a9b02244c1ae00a","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-int32-decimal","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"24b740aac5dd539de0a5f16d4b965e6c8aeaf68f8907afaa2a9b02244c1ae00a","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int32-with-null-pages","column_order_count":1,"created_by":"parquet-mr version 1.13.0-SNAPSHOT (build 433de8df33fcf31927f7b51456be9f53e64d48b9)","created_by_present":true,"file":"data/int32_with_null_pages.parquet","footer_length":265,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7","size":3829} +{"case_id":"apache-int32-with-null-pages","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"0720e57f","deprecated_min_hex":"c664a180","distinct_count":null,"file":"data/int32_with_null_pages.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0720e57f","min_value_hex":"c664a180","nan_count":null,"null_count":"275","num_values":"1000","path":["int32_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a","capability_id":"wire.column-order.type","case_id":"apache-int32-with-null-pages","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"96a7af0529933b14d5e09b713cb7224bc9a241ecf621dd718be7c8f205115c97","capability_id":"wire.statistics.counts","case_id":"apache-int32-with-null-pages","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"96a7af0529933b14d5e09b713cb7224bc9a241ecf621dd718be7c8f205115c97","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f269313aafeb3a7f07de5b30c207665c861a665226ed0507a3a93d84d543c603","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-int32-with-null-pages","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f269313aafeb3a7f07de5b30c207665c861a665226ed0507a3a93d84d543c603","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"c379fc9587e1ebc82c952dcbc0ba1fe0f11ace06e29cf99655bdf3ee54493e1a","capability_id":"wire.statistics.modern-bounds","case_id":"apache-int32-with-null-pages","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c379fc9587e1ebc82c952dcbc0ba1fe0f11ace06e29cf99655bdf3ee54493e1a","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int64-decimal","column_order_count":null,"created_by":"parquet-mr version 1.8.2 (build c6522788629e590a53eb79874b95f6c3ff11f16c)","created_by_present":true,"file":"data/int64_decimal.parquet","footer_length":338,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56","size":591} +{"case_id":"apache-int64-decimal","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"6009000000000000","deprecated_min_hex":"6400000000000000","distinct_count":null,"file":"data/int64_decimal.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT64","precision":10,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"24","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"6646160402e91c23877fba9f8bd2c5ea5e39c8ce29afa54b3a705a2a270c96e7","capability_id":"wire.statistics.counts","case_id":"apache-int64-decimal","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"6646160402e91c23877fba9f8bd2c5ea5e39c8ce29afa54b3a705a2a270c96e7","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"6884fc4032424ea758097f8160a9ad6affbf9b3cdc8b5e35efad5991af35b862","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-int64-decimal","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"6884fc4032424ea758097f8160a9ad6affbf9b3cdc8b5e35efad5991af35b862","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-int96-timestamp-order","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 8931c1c55f1fba399dd75139f75bcde0b84137c0)","created_by_present":true,"file":"data/int96_timestamp_order.parquet","footer_length":271,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"e35f8748d286a729e719a01c5411f81c79802d61a55046d5cf9a663918a14644","size":427} +{"case_id":"apache-int96-timestamp-order","column_order":{"field_id":3,"header_hex":"3c","state":"UNKNOWN","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/int96_timestamp_order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT96","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000000000000008d3d2500","min_value_hex":"7b00000000000000403b2500","nan_count":null,"null_count":"0","num_values":"4","path":["ts"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"739a3ea3568d8f0d83adec173cf04047fc4be1d90016d19b9dbce26bef0617c3","capability_id":"wire.column-order.empty","case_id":"apache-int96-timestamp-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"739a3ea3568d8f0d83adec173cf04047fc4be1d90016d19b9dbce26bef0617c3","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"3a8cd38cc7dd0fc83709b5f27f34ae29f534254e4f4f11160dbaac80b666d740","capability_id":"wire.statistics.counts","case_id":"apache-int96-timestamp-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"3a8cd38cc7dd0fc83709b5f27f34ae29f534254e4f4f11160dbaac80b666d740","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"55d8f84db701c73eee56e21bfe0c23029bb811dd7883d8e8b4c4a974faf7f920","capability_id":"wire.statistics.modern-bounds","case_id":"apache-int96-timestamp-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"55d8f84db701c73eee56e21bfe0c23029bb811dd7883d8e8b4c4a974faf7f920","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-json","column_order_count":1,"created_by":"parquet-mr version 1.18.0-SNAPSHOT (build 1a9e455655604acf09cdd45b4e2958661d38281c)","created_by_present":true,"file":"data/json.parquet","footer_length":270,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5","size":402} +{"case_id":"apache-json","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/json.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2261223a317d","min_value_hex":"5b312c6e756c6c2c335d","nan_count":null,"null_count":"1","num_values":"4","path":["json_field"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04","capability_id":"wire.column-order.type","case_id":"apache-json","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"a45e1a3a551262956760db3ac3ca3679b03cc7d28e7c5eced2d425768efcd89f","capability_id":"wire.statistics.counts","case_id":"apache-json","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"a45e1a3a551262956760db3ac3ca3679b03cc7d28e7c5eced2d425768efcd89f","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"998ad186ca9f239a8e299a83ae348bb49c1396ff6c1d365f5ce547a373dae261","capability_id":"wire.statistics.modern-bounds","case_id":"apache-json","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"998ad186ca9f239a8e299a83ae348bb49c1396ff6c1d365f5ce547a373dae261","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-nan-in-stats","column_order_count":1,"created_by":"parquet-cpp version 1.3.2-SNAPSHOT","created_by_present":true,"file":"data/nan_in_stats.parquet","footer_length":156,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337","size":329} +{"case_id":"apache-nan-in-stats","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"000000000000f87f","deprecated_min_hex":"000000000000f03f","distinct_count":null,"file":"data/nan_in_stats.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"000000000000f87f","min_value_hex":"000000000000f03f","nan_count":null,"null_count":"0","num_values":"2","path":["x"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7","capability_id":"wire.column-order.type","case_id":"apache-nan-in-stats","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"b5b0a51b593f25130fdbea68ed10ae7f5c9516e487ab95abcbc83d5511280c3f","capability_id":"wire.statistics.counts","case_id":"apache-nan-in-stats","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"b5b0a51b593f25130fdbea68ed10ae7f5c9516e487ab95abcbc83d5511280c3f","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f0da4b9fd47eb868f71749b2d701acda9f0dbea428eb1a15fac4406142b4b773","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-nan-in-stats","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f0da4b9fd47eb868f71749b2d701acda9f0dbea428eb1a15fac4406142b4b773","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"78fd4d2063e9b4c2afa8ee6c63747e8ac9de742005e8cb6f6463de0f7515e83e","capability_id":"wire.statistics.modern-bounds","case_id":"apache-nan-in-stats","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"78fd4d2063e9b4c2afa8ee6c63747e8ac9de742005e8cb6f6463de0f7515e83e","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-rle-boolean-encoding","column_order_count":null,"created_by":null,"created_by_present":false,"file":"data/rle_boolean_encoding.parquet","footer_length":111,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76","size":192} +{"case_id":"apache-rle-boolean-encoding","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":"01","deprecated_min_hex":"00","distinct_count":null,"file":"data/rle_boolean_encoding.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"6","num_values":"68","path":["datatype_boolean"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"ef968b5ad71e18e974b3111d67676f55cad8c7cf38e614d932e4d756e72f6e0b","capability_id":"wire.statistics.counts","case_id":"apache-rle-boolean-encoding","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ef968b5ad71e18e974b3111d67676f55cad8c7cf38e614d932e4d756e72f6e0b","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"88f5317fa10dd3c9998b6c1702b2a2c6363323bc3ed0b079cbb732ff68988047","capability_id":"wire.statistics.deprecated-bounds","case_id":"apache-rle-boolean-encoding","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"88f5317fa10dd3c9998b6c1702b2a2c6363323bc3ed0b079cbb732ff68988047","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"f448728e40f359dd8454cacaff7444fa8563ba8abde10cfd2191d6df49ae6594","capability_id":"wire.statistics.modern-bounds","case_id":"apache-rle-boolean-encoding","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"f448728e40f359dd8454cacaff7444fa8563ba8abde10cfd2191d6df49ae6594","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"apache-single-nan","column_order_count":1,"created_by":"parquet-cpp version 1.5.1-SNAPSHOT","created_by_present":true,"file":"data/single_nan.parquet","footer_length":567,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"ea3371c44ed1794843a2f529888120537f68aedcb80d6fbe32cea1003ab5769e","size":660} +{"case_id":"apache-single-nan","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"data/single_nan.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"1","num_values":"1","path":["mycol"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d","capability_id":"wire.column-order.type","case_id":"apache-single-nan","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"10249d7c339fda733aa6c14216562935ad7e1394329dc47f4b556ecca362a384","capability_id":"wire.statistics.counts","case_id":"apache-single-nan","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"10249d7c339fda733aa6c14216562935ad7e1394329dc47f4b556ecca362a384","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":null,"capability_id":"semantic.type-order","case_id":"atomic-bound-family","detail":"The reviewed capability matrix marks this result unsupported.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":null,"record":"case_result","schema_version":2,"status":"UNSUPPORTED"} diff --git a/test/conformance/n6/evidence/raw-java-generated.normalized.jsonl b/test/conformance/n6/evidence/raw-java-generated.normalized.jsonl new file mode 100644 index 0000000..50d5b75 --- /dev/null +++ b/test/conformance/n6/evidence/raw-java-generated.normalized.jsonl @@ -0,0 +1,82 @@ +{"capabilities_sha256":"50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419","corpus_manifest_sha256":"10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991","evidence_id":"normalized-raw-java-generated","evidence_schema_sha256":"5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31","fixture_manifest_sha256":"670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25","plan_sha256":"15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304","producer":"n6-raw-java","producer_version":"parquet-2.13-raw-footer-v3","record":"run","schema_version":2,"source_revision":"c47e2a66e88943fc46fde1b028a9432f14fdf5c0","toolchain_sha256":"8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629","unsupported_cases":[]} +{"case_id":"julia-writer-type-order","column_order_count":8,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-type-order.parquet","footer_length":1195,"leaf_count":8,"record":"file","row_group_count":2,"schema_version":2,"sha256":"333a9c3042992ba5f14c97db59c42fc98d4093bebcbab942cfcdfdedc43adf8a","size":1884} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"09000000","min_value_hex":"1cfbffff","nan_count":null,"null_count":"1","num_values":"3","path":["signed"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":32,"converted_type":"UINT_32","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":false,"logical_type":"INTEGER","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ffffffff","min_value_hex":"00000000","nan_count":null,"null_count":"0","num_values":"3","path":["unsigned"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ff","min_value_hex":"00cf","nan_count":null,"null_count":"1","num_values":"3","path":["raw"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a","min_value_hex":"61","nan_count":null,"null_count":"1","num_values":"3","path":["text"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT32","precision":9,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":"00000000","min_value_hex":"6fa0feff","nan_count":null,"null_count":"1","num_values":"3","path":["decimal"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":"DATE","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DATE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"00000000","min_value_hex":"219cffff","nan_count":null,"null_count":"1","num_values":"3","path":["dates"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"1","num_values":"3","path":["flag"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":3},"max_value_hex":"ff0000","min_value_hex":"00ff28","nan_count":null,"null_count":"0","num_values":"3","path":["fixed"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"ac050000","min_value_hex":"f9ffffff","nan_count":null,"null_count":"0","num_values":"3","path":["signed"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":32,"converted_type":"UINT_32","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":false,"logical_type":"INTEGER","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"86000000","min_value_hex":"01000000","nan_count":null,"null_count":"0","num_values":"3","path":["unsigned"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"80","min_value_hex":"","nan_count":null,"null_count":"0","num_values":"3","path":["raw"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a7a","min_value_hex":"610062","nan_count":null,"null_count":"0","num_values":"3","path":["text"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":4,"leaf_schema":{"bit_width":null,"converted_type":"DECIMAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DECIMAL","physical_type":"INT32","precision":9,"scale":2,"time_unit":null,"type_length":null},"max_value_hex":"9f860100","min_value_hex":"ffffffff","nan_count":null,"null_count":"0","num_values":"3","path":["decimal"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":5,"leaf_schema":{"bit_width":null,"converted_type":"DATE","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"DATE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7ab90000","min_value_hex":"ffffffff","nan_count":null,"null_count":"0","num_values":"3","path":["dates"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":6,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"3","path":["flag"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-type-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-type-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":7,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":3},"max_value_hex":"800000","min_value_hex":"010203","nan_count":null,"null_count":"0","num_values":"3","path":["fixed"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"ef0a946f6ec243d4d7652fec738bea944a1cba488ff131fc813daf6c02d2c641","capability_id":"wire.column-order.type","case_id":"julia-writer-type-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ef0a946f6ec243d4d7652fec738bea944a1cba488ff131fc813daf6c02d2c641","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"2610efdc7af17a63429a1b4b04c6e3429774c0ef4e77c21e40dde2685925b662","capability_id":"wire.statistics.counts","case_id":"julia-writer-type-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"2610efdc7af17a63429a1b4b04c6e3429774c0ef4e77c21e40dde2685925b662","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"d0a89674be145b02f6548d2f76ec6180ad183a7b964451efbd0554803f0cf842","capability_id":"wire.statistics.exactness","case_id":"julia-writer-type-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d0a89674be145b02f6548d2f76ec6180ad183a7b964451efbd0554803f0cf842","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"34fe7df73b17c8e015fb68b94fd3cbd3372f1eea5f839606553cd35e083dac40","capability_id":"wire.statistics.modern-bounds","case_id":"julia-writer-type-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"34fe7df73b17c8e015fb68b94fd3cbd3372f1eea5f839606553cd35e083dac40","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-ieee-order","column_order_count":3,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-ieee-order.parquet","footer_length":531,"leaf_count":3,"record":"file","row_group_count":2,"schema_version":2,"sha256":"8782589a7b6bfcb576acf04e0bc15cef156b931497f63a5623c7cb2b4355188c","size":907} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"007c","min_value_hex":"00bc","nan_count":"1","null_count":"1","num_values":"6","path":["half"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0000807f","min_value_hex":"000080bf","nan_count":"1","null_count":"1","num_values":"6","path":["single"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"000000000000f07f","min_value_hex":"000000000000f0bf","nan_count":"1","null_count":"1","num_values":"6","path":["double"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"FLOAT16","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":2},"max_value_hex":"107e","min_value_hex":"20fe","nan_count":"6","null_count":"0","num_values":"6","path":["half"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"FLOAT","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"2222e27f","min_value_hex":"2143c5ff","nan_count":"6","null_count":"0","num_values":"6","path":["single"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-ieee-order","column_order":{"field_id":2,"header_hex":"2c","state":"IEEE_754_TOTAL_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-ieee-order.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"DOUBLE","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"452301000000f87f","min_value_hex":"220000000000f8ff","nan_count":"6","null_count":"0","num_values":"6","path":["double"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"1dafbdaddfc0a9f5d2c446bd8330b79a9b81c0e9daa5c1857c49d23632c19ca9","capability_id":"wire.column-order.ieee","case_id":"julia-writer-ieee-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"1dafbdaddfc0a9f5d2c446bd8330b79a9b81c0e9daa5c1857c49d23632c19ca9","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"7dace151af90326c6026b487b88b82ca035da6d5c78fd2785ed424913cac8f39","capability_id":"wire.statistics.counts","case_id":"julia-writer-ieee-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7dace151af90326c6026b487b88b82ca035da6d5c78fd2785ed424913cac8f39","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"ec85e35f110db9e727099e24738bf86c39519054df6a98de42697aec3b3d5dc1","capability_id":"wire.statistics.exactness","case_id":"julia-writer-ieee-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"ec85e35f110db9e727099e24738bf86c39519054df6a98de42697aec3b3d5dc1","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"7d46ab56682db03224c3ce4f6715d322385625c1260d16af80c75e479315e81c","capability_id":"wire.statistics.modern-bounds","case_id":"julia-writer-ieee-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"7d46ab56682db03224c3ce4f6715d322385625c1260d16af80c75e479315e81c","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"25a04c84a4c217fd36f1f5b38770807e85265fc20521744ebdd70f44b743342d","capability_id":"wire.statistics.nan-count","case_id":"julia-writer-ieee-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"25a04c84a4c217fd36f1f5b38770807e85265fc20521744ebdd70f44b743342d","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-undefined-order","column_order_count":3,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-undefined-order.parquet","footer_length":274,"leaf_count":3,"record":"file","row_group_count":1,"schema_version":2,"sha256":"ce53d8ec5024f2694bc5a50782c918f1368b39ae4d8c94bc80791e048bb28648","size":454} +{"case_id":"julia-writer-undefined-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-undefined-order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"INTERVAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"INTERVAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":12},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"4","path":["first"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-undefined-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-undefined-order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":"INTERVAL","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"INTERVAL","physical_type":"FIXED_LEN_BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":12},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"2","num_values":"4","path":["second"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-undefined-order","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-undefined-order.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"UNKNOWN","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"4","num_values":"4","path":["unknown"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"88fea0ed126fc9774d55e8bd4969fbb7f0060462f6c330d62c3935fc520895b5","capability_id":"wire.column-order.type","case_id":"julia-writer-undefined-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"88fea0ed126fc9774d55e8bd4969fbb7f0060462f6c330d62c3935fc520895b5","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"2da1417c6bde7ce253b654ef1f28785d65a8ce6d58f00160908c178e901afaf5","capability_id":"wire.statistics.counts","case_id":"julia-writer-undefined-order","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"2da1417c6bde7ce253b654ef1f28785d65a8ce6d58f00160908c178e901afaf5","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-statistics-disabled","column_order_count":null,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-statistics-disabled.parquet","footer_length":285,"leaf_count":2,"record":"file","row_group_count":2,"schema_version":2,"sha256":"b0be13b0030888a41c82de57fafa987b29997352cd923c7542539b34a39141fd","size":494} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["number"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["text"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["number"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-statistics-disabled","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-statistics-disabled.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["text"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-oversized-bounds","column_order_count":1,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-oversized-bounds.parquet","footer_length":141,"leaf_count":1,"record":"file","row_group_count":1,"schema_version":2,"sha256":"0dc68216f7dd5b1ef425b59e398553af71f254fe249c7dba30de82d25d491c9a","size":12494} +{"case_id":"julia-writer-oversized-bounds","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-oversized-bounds.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"3","path":["value"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"d7f2bda960a5ce3e44aaf94949136e1c5a58547e6a752f8b36854fe60b210f9e","capability_id":"wire.column-order.type","case_id":"julia-writer-oversized-bounds","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"d7f2bda960a5ce3e44aaf94949136e1c5a58547e6a752f8b36854fe60b210f9e","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"9cf8a360d6631b0ca771386c61cd674e1ccd66835af4ac1f8d06ff267b24d6f3","capability_id":"wire.statistics.counts","case_id":"julia-writer-oversized-bounds","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"9cf8a360d6631b0ca771386c61cd674e1ccd66835af4ac1f8d06ff267b24d6f3","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-writer-nested-row-groups","column_order_count":4,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-writer-nested-row-groups.parquet","footer_length":673,"leaf_count":4,"record":"file","row_group_count":2,"schema_version":2,"sha256":"9a705db14a5c00260717bc907a916303ce297a039c03fb0f3ae981022e763d49","size":1025} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"62","min_value_hex":"61","nan_count":null,"null_count":"1","num_values":"3","path":["tag"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"a4000000","min_value_hex":"02000000","nan_count":null,"null_count":"1","num_values":"3","path":["rows","id"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"3","num_values":"3","path":["rows","items","list","element"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"3","path":["flag"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a","min_value_hex":"63","nan_count":null,"null_count":"1","num_values":"3","path":["tag"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"05000000","min_value_hex":"03000000","nan_count":null,"null_count":"0","num_values":"3","path":["rows","id"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":2,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"0a000000","min_value_hex":"ffffffff","nan_count":null,"null_count":"2","num_values":"6","path":["rows","items","list","element"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-writer-nested-row-groups","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-writer-nested-row-groups.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":3,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"BOOLEAN","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"01","min_value_hex":"00","nan_count":null,"null_count":"0","num_values":"3","path":["flag"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"actual_sha256":"c3afbf4b4e641914adc0ca2e0b829cd0422c65804522f2f1f7e7a8aacf479bf7","capability_id":"wire.column-order.type","case_id":"julia-writer-nested-row-groups","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"c3afbf4b4e641914adc0ca2e0b829cd0422c65804522f2f1f7e7a8aacf479bf7","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"189a107784eacf2541116edbecb2d88559766bd9cce50f8f5dca5fbd7326aa27","capability_id":"wire.statistics.counts","case_id":"julia-writer-nested-row-groups","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"189a107784eacf2541116edbecb2d88559766bd9cce50f8f5dca5fbd7326aa27","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"5c3e2d05d6733defe16898006f2ab732213d5c9117f295eec11a8b0d267456a6","capability_id":"wire.statistics.exactness","case_id":"julia-writer-nested-row-groups","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"5c3e2d05d6733defe16898006f2ab732213d5c9117f295eec11a8b0d267456a6","record":"case_result","schema_version":2,"status":"PASS"} +{"actual_sha256":"932d400f467a09a54db5aaac748ebd3134fcb7f249bf6aa082f82f6e4679c85d","capability_id":"wire.statistics.modern-bounds","case_id":"julia-writer-nested-row-groups","detail":"Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.","digest_contract":"n6-capability-result-sha256-v1","expected_sha256":"932d400f467a09a54db5aaac748ebd3134fcb7f249bf6aa082f82f6e4679c85d","record":"case_result","schema_version":2,"status":"PASS"} +{"case_id":"julia-reader-untrusted-producer","column_order_count":2,"created_by":"parquet-mr version 1.7.0 (build n6)","created_by_present":true,"file":"generated/julia-reader-untrusted-producer.parquet","footer_length":370,"leaf_count":2,"record":"file","row_group_count":2,"schema_version":2,"sha256":"57f7c7b1d350fc0f84bb1de006aa7ed992194f1a12eea28a506b97e736039b93","size":575} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7a","min_value_hex":"61","nan_count":null,"null_count":"1","num_values":"3","path":["binary"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"09000000","deprecated_min_hex":"f6ffffff","distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"1","num_values":"3","path":["signed"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"UTF8","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"STRING","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7461696c","min_value_hex":"6d6964646c65","nan_count":null,"null_count":"0","num_values":"3","path":["binary"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-untrusted-producer","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":"b6000000","deprecated_min_hex":"9dffffff","distinct_count":null,"file":"generated/julia-reader-untrusted-producer.parquet","has_statistics":true,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":1,"leaf_schema":{"bit_width":null,"converted_type":null,"crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"NONE","physical_type":"INT32","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":"0","num_values":"3","path":["signed"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-absent","column_order_count":null,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-reader-no-pruning-absent.parquet","footer_length":177,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"a817b4c27f01744c853aa09399c002127463940eb9a94f5b31b7fa57b33e2927","size":321} +{"case_id":"julia-reader-no-pruning-absent","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-absent.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-absent","column_order":{"field_id":null,"header_hex":null,"state":"ABSENT","wire_type":null},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-absent.parquet","has_statistics":false,"is_max_value_exact":null,"is_min_value_exact":null,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":null,"min_value_hex":null,"nan_count":null,"null_count":null,"num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-trusted","column_order_count":1,"created_by":"parquet-mr version 1.10.0 (build n6)","created_by_present":true,"file":"generated/julia-reader-no-pruning-trusted.parquet","footer_length":250,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"b9e499b621048f4e72d25cef732e800bfe380ee03d0eedce46de67602bf2767f","size":394} +{"case_id":"julia-reader-no-pruning-trusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-trusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2276616c7565223a32337d","min_value_hex":"5b312c322c335d","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-trusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-trusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b226e6573746564223a747275657d","min_value_hex":"227461696c22","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-untrusted","column_order_count":1,"created_by":"parquet-mr version 1.7.0 (build n6)","created_by_present":true,"file":"generated/julia-reader-no-pruning-untrusted.parquet","footer_length":249,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"22b92a7f1139a6d7ce8f33a5440b4891db9eae323c5e3d0c620969d3f5761771","size":393} +{"case_id":"julia-reader-no-pruning-untrusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-untrusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2276616c7565223a32337d","min_value_hex":"5b312c322c335d","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-untrusted","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-untrusted.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b226e6573746564223a747275657d","min_value_hex":"227461696c22","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-oversized","column_order_count":1,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-reader-no-pruning-oversized.parquet","footer_length":16594,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"1e416cfdbb7b25c5a7e0013845a24ef2ff28e2948b326bacb9eeb38b8e7862a9","size":16738} +{"case_id":"julia-reader-no-pruning-oversized","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-oversized.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","min_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-oversized","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-oversized.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","min_value_hex":"7878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-invalid","column_order_count":1,"created_by":"Parquet.jl version 1.0.0-DEV","created_by_present":true,"file":"generated/julia-reader-no-pruning-invalid.parquet","footer_length":235,"leaf_count":1,"record":"file","row_group_count":2,"schema_version":2,"sha256":"360c5e75bd2fa90a28e05ba46990f44f98b5e32452555d7c7f84ce5db7454d20","size":379} +{"case_id":"julia-reader-no-pruning-invalid","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-invalid.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b2276616c7565223a32337d","min_value_hex":"000000","nan_count":null,"null_count":"1","num_values":"3","path":["json"],"record":"column_statistics","row_group":0,"schema_version":2,"unknown_statistics_field_ids":[]} +{"case_id":"julia-reader-no-pruning-invalid","column_order":{"field_id":1,"header_hex":"1c","state":"TYPE_ORDER","wire_type":12},"deprecated_max_hex":null,"deprecated_min_hex":null,"distinct_count":null,"file":"generated/julia-reader-no-pruning-invalid.parquet","has_statistics":true,"is_max_value_exact":true,"is_min_value_exact":true,"leaf":0,"leaf_schema":{"bit_width":null,"converted_type":"JSON","crs":null,"geography_algorithm":null,"is_adjusted_to_utc":null,"is_signed":null,"logical_type":"JSON","physical_type":"BYTE_ARRAY","precision":null,"scale":null,"time_unit":null,"type_length":null},"max_value_hex":"7b226e6573746564223a747275657d","min_value_hex":"000000","nan_count":null,"null_count":"0","num_values":"3","path":["json"],"record":"column_statistics","row_group":1,"schema_version":2,"unknown_statistics_field_ids":[]} diff --git a/test/conformance/n6/fixtures.toml b/test/conformance/n6/fixtures.toml new file mode 100644 index 0000000..e2a5641 --- /dev/null +++ b/test/conformance/n6/fixtures.toml @@ -0,0 +1,573 @@ +manifest_version = 1 +authority = "parquet-testing" +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +checksum_manifest = "corpus-files.sha256" +status_values = ["verified", "planned", "unsupported"] +output_identity_statuses = ["planned", "verified"] +generation_contract_version = 1 +default_digest_contract = "n6-capability-result-sha256-v1" +digest_contracts = ["n6-capability-result-sha256-v1", "n6-no-pruning-trace-sha256-v1"] + +[[fixture]] +id = "apache-alltypes-dictionary" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/alltypes_dictionary.parquet" +sha256 = "7b58c33503858c533e1521b3022b85a0de23e5a144420d7a3c1c426929e5f6fb" +size = 1698 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 11 +normalized_record_count = 12 +capabilities = ["read.logical-values"] +expected_unsupported = [] + +[[generated_case]] +id = "julia-writer-type-order" +status = "verified" +source_kind = "julia-writer-generated" +authority = "parquet-jl" +output_file = "generated/julia-writer-type-order.parquet" +output_identity_status = "verified" +output_sha256 = "333a9c3042992ba5f14c97db59c42fc98d4093bebcbab942cfcdfdedc43adf8a" +output_size = 1884 +generator_profile = "writer-type-order-v1" +generator_seed = 6001 +variant_id = "type-order" +mutation = { kind = "none" } +comparison_group = "" +digest_contract = "n6-capability-result-sha256-v1" +row_group_count = 2 +leaf_count = 8 +normalized_record_count = 17 +capabilities = ["write.type-order", "wire.column-order.type", "wire.statistics.modern-bounds", "wire.statistics.exactness", "wire.statistics.counts", "semantic.type-order", "semantic.logical-order", "read.logical-values"] +expected_unsupported = [] +description = "Signed, unsigned, byte-wise, decimal, temporal, and boolean extrema." + +[[generated_case]] +id = "julia-writer-ieee-order" +status = "verified" +source_kind = "julia-writer-generated" +authority = "parquet-jl" +output_file = "generated/julia-writer-ieee-order.parquet" +output_identity_status = "verified" +output_sha256 = "8782589a7b6bfcb576acf04e0bc15cef156b931497f63a5623c7cb2b4355188c" +output_size = 907 +generator_profile = "writer-ieee-order-v1" +generator_seed = 6002 +variant_id = "ieee-order" +mutation = { kind = "none" } +comparison_group = "" +digest_contract = "n6-capability-result-sha256-v1" +row_group_count = 2 +leaf_count = 3 +normalized_record_count = 7 +capabilities = ["write.type-order", "wire.column-order.ieee", "wire.statistics.modern-bounds", "wire.statistics.exactness", "wire.statistics.counts", "wire.statistics.nan-count", "semantic.ieee-total-order", "semantic.count-state", "read.logical-values"] +expected_unsupported = [] +description = "Float16, Float32, and Float64 zeros, finite values, infinities, and NaN payloads." + +[[generated_case]] +id = "julia-writer-undefined-order" +status = "verified" +source_kind = "julia-writer-generated" +authority = "parquet-jl" +output_file = "generated/julia-writer-undefined-order.parquet" +output_identity_status = "verified" +output_sha256 = "ce53d8ec5024f2694bc5a50782c918f1368b39ae4d8c94bc80791e048bb28648" +output_size = 454 +generator_profile = "writer-undefined-order-v1" +generator_seed = 6003 +variant_id = "undefined-order" +mutation = { kind = "none" } +comparison_group = "" +digest_contract = "n6-capability-result-sha256-v1" +row_group_count = 1 +leaf_count = 3 +normalized_record_count = 4 +capabilities = ["write.type-order", "wire.column-order.type", "wire.statistics.counts", "semantic.type-order"] +expected_unsupported = [] +description = "Undefined-order leaves retain counts and omit bounds." + +[[generated_case]] +id = "julia-writer-statistics-disabled" +status = "verified" +source_kind = "julia-writer-generated" +authority = "parquet-jl" +output_file = "generated/julia-writer-statistics-disabled.parquet" +output_identity_status = "verified" +output_sha256 = "b0be13b0030888a41c82de57fafa987b29997352cd923c7542539b34a39141fd" +output_size = 494 +generator_profile = "writer-statistics-disabled-v1" +generator_seed = 6004 +variant_id = "statistics-disabled" +mutation = { kind = "none" } +comparison_group = "" +digest_contract = "n6-capability-result-sha256-v1" +row_group_count = 2 +leaf_count = 2 +normalized_record_count = 5 +capabilities = ["write.statistics-disabled", "read.logical-values"] +expected_unsupported = [] +description = "statistics=false preserves the old body and omits statistics and column_orders." + +[[generated_case]] +id = "julia-writer-oversized-bounds" +status = "verified" +source_kind = "julia-writer-generated" +authority = "parquet-jl" +output_file = "generated/julia-writer-oversized-bounds.parquet" +output_identity_status = "verified" +output_sha256 = "0dc68216f7dd5b1ef425b59e398553af71f254fe249c7dba30de82d25d491c9a" +output_size = 12494 +generator_profile = "writer-oversized-bounds-v1" +generator_seed = 6005 +variant_id = "one-byte-over" +mutation = { kind = "none" } +comparison_group = "" +digest_contract = "n6-capability-result-sha256-v1" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["write.type-order", "write.statistics-limit", "wire.column-order.type", "wire.statistics.counts"] +expected_unsupported = [] +description = "One-byte-over bounds are both omitted while exact counts remain." + +[[generated_case]] +id = "julia-writer-nested-row-groups" +status = "verified" +source_kind = "julia-writer-generated" +authority = "parquet-jl" +output_file = "generated/julia-writer-nested-row-groups.parquet" +output_identity_status = "verified" +output_sha256 = "9a705db14a5c00260717bc907a916303ce297a039c03fb0f3ae981022e763d49" +output_size = 1025 +generator_profile = "writer-nested-row-groups-v1" +generator_seed = 6006 +variant_id = "nested-row-groups" +mutation = { kind = "none" } +comparison_group = "" +digest_contract = "n6-capability-result-sha256-v1" +row_group_count = 2 +leaf_count = 4 +normalized_record_count = 9 +capabilities = ["write.type-order", "wire.column-order.type", "wire.statistics.modern-bounds", "wire.statistics.exactness", "wire.statistics.counts", "semantic.type-order", "semantic.count-state", "read.logical-values"] +expected_unsupported = [] +description = "Nested leaf-entry counts are independent of row-group row counts." + +[[generated_case]] +id = "julia-reader-untrusted-producer" +status = "verified" +source_kind = "metadata-mutation-generated" +authority = "parquet-jl" +output_file = "generated/julia-reader-untrusted-producer.parquet" +output_identity_status = "verified" +output_sha256 = "57f7c7b1d350fc0f84bb1de006aa7ed992194f1a12eea28a506b97e736039b93" +output_size = 575 +generator_profile = "reader-untrusted-producer-v1" +generator_seed = 6007 +variant_id = "parquet-mr-1.7.0" +mutation = { kind = "created-by", created_by = "parquet-mr version 1.7.0 (build n6)" } +comparison_group = "" +digest_contract = "n6-capability-result-sha256-v1" +row_group_count = 2 +leaf_count = 2 +normalized_record_count = 5 +capabilities = ["semantic.producer-trust", "compat.legacy-statistics", "read.logical-values"] +expected_unsupported = [] +description = "Affected producer identities suppress bounds but preserve valid counts and values." + +[[generated_case]] +id = "julia-reader-no-pruning-absent" +status = "verified" +source_kind = "metadata-mutation-generated" +authority = "parquet-jl" +output_file = "generated/julia-reader-no-pruning-absent.parquet" +output_identity_status = "verified" +output_sha256 = "a817b4c27f01744c853aa09399c002127463940eb9a94f5b31b7fa57b33e2927" +output_size = 321 +generator_profile = "reader-no-pruning-v1" +generator_seed = 6008 +variant_id = "absent" +mutation = { kind = "statistics-state", statistics_state = "absent" } +comparison_group = "julia-reader-no-pruning-v1" +digest_contract = "n6-no-pruning-trace-sha256-v1" +row_group_count = 2 +leaf_count = 1 +normalized_record_count = 3 +capabilities = ["read.no-pruning", "read.logical-values"] +expected_unsupported = [] +description = "Statistics and column orders are absent." + +[[generated_case]] +id = "julia-reader-no-pruning-trusted" +status = "verified" +source_kind = "metadata-mutation-generated" +authority = "parquet-jl" +output_file = "generated/julia-reader-no-pruning-trusted.parquet" +output_identity_status = "verified" +output_sha256 = "b9e499b621048f4e72d25cef732e800bfe380ee03d0eedce46de67602bf2767f" +output_size = 394 +generator_profile = "reader-no-pruning-v1" +generator_seed = 6008 +variant_id = "trusted" +mutation = { kind = "statistics-state", statistics_state = "trusted", created_by = "parquet-mr version 1.10.0 (build n6)" } +comparison_group = "julia-reader-no-pruning-v1" +digest_contract = "n6-no-pruning-trace-sha256-v1" +row_group_count = 2 +leaf_count = 1 +normalized_record_count = 3 +capabilities = ["read.no-pruning", "read.logical-values"] +expected_unsupported = [] +description = "Trusted exact statistics are present." + +[[generated_case]] +id = "julia-reader-no-pruning-untrusted" +status = "verified" +source_kind = "metadata-mutation-generated" +authority = "parquet-jl" +output_file = "generated/julia-reader-no-pruning-untrusted.parquet" +output_identity_status = "verified" +output_sha256 = "22b92a7f1139a6d7ce8f33a5440b4891db9eae323c5e3d0c620969d3f5761771" +output_size = 393 +generator_profile = "reader-no-pruning-v1" +generator_seed = 6008 +variant_id = "producer-untrusted" +mutation = { kind = "statistics-state", statistics_state = "producer-untrusted", created_by = "parquet-mr version 1.7.0 (build n6)" } +comparison_group = "julia-reader-no-pruning-v1" +digest_contract = "n6-no-pruning-trace-sha256-v1" +row_group_count = 2 +leaf_count = 1 +normalized_record_count = 3 +capabilities = ["read.no-pruning", "read.logical-values"] +expected_unsupported = [] +description = "Producer policy makes otherwise valid bounds untrusted." + +[[generated_case]] +id = "julia-reader-no-pruning-oversized" +status = "verified" +source_kind = "metadata-mutation-generated" +authority = "parquet-jl" +output_file = "generated/julia-reader-no-pruning-oversized.parquet" +output_identity_status = "verified" +output_sha256 = "1e416cfdbb7b25c5a7e0013845a24ef2ff28e2948b326bacb9eeb38b8e7862a9" +output_size = 16738 +generator_profile = "reader-no-pruning-v1" +generator_seed = 6008 +variant_id = "oversized" +mutation = { kind = "statistics-state", statistics_state = "oversized", bound_bytes = 4097 } +comparison_group = "julia-reader-no-pruning-v1" +digest_contract = "n6-no-pruning-trace-sha256-v1" +row_group_count = 2 +leaf_count = 1 +normalized_record_count = 3 +capabilities = ["read.no-pruning", "read.logical-values"] +expected_unsupported = [] +description = "Both variable bounds are one byte over the interpretation limit." + +[[generated_case]] +id = "julia-reader-no-pruning-invalid" +status = "verified" +source_kind = "metadata-mutation-generated" +authority = "parquet-jl" +output_file = "generated/julia-reader-no-pruning-invalid.parquet" +output_identity_status = "verified" +output_sha256 = "360c5e75bd2fa90a28e05ba46990f44f98b5e32452555d7c7f84ce5db7454d20" +output_size = 379 +generator_profile = "reader-no-pruning-v1" +generator_seed = 6008 +variant_id = "semantically-unusable" +mutation = { kind = "statistics-state", statistics_state = "semantically-unusable", field = "min_value", value_hex = "000000" } +comparison_group = "julia-reader-no-pruning-v1" +digest_contract = "n6-no-pruning-trace-sha256-v1" +row_group_count = 2 +leaf_count = 1 +normalized_record_count = 3 +capabilities = ["read.no-pruning", "read.logical-values"] +expected_unsupported = [] +description = "A JSON BYTE_ARRAY lower bound contains the semantically invalid bytes 000000." + +[[fixture]] +id = "apache-alltypes-plain" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/alltypes_plain.parquet" +sha256 = "12a618d20a59ee0967fef45e7ec1ff6d451e724838edc1bbeac780ca15e8fcc4" +size = 1851 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 11 +normalized_record_count = 12 +capabilities = ["read.logical-values"] +expected_unsupported = [] + +[[fixture]] +id = "apache-binary" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/binary.parquet" +sha256 = "b48b756e48a13f58e1234a8588c507a06a7a9bcdfb63994c86fe19d22864be8b" +size = 478 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.logical-order", "read.logical-values", "read.statistics-metadata", "read.parquet-metadata-view"] +expected_unsupported = [] + +[[fixture]] +id = "apache-binary-truncated-min-max" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/binary_truncated_min_max.parquet" +sha256 = "94a1e9ef0cd5104168c1e80480fac8918a962a355d9ab33bca3e13ff4402b201" +size = 3070 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 6 +normalized_record_count = 7 +capabilities = ["wire.column-order.type", "wire.statistics.modern-bounds", "wire.statistics.exactness", "wire.statistics.counts", "read.statistics-metadata", "read.parquet-metadata-view"] +expected_unsupported = [] + +[[fixture]] +id = "apache-bson" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/bson.parquet" +sha256 = "44b503ac1ecb70627b29fbce2d3109cd161afe5c0f72b556978b454fc64cb129" +size = 412 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.logical-order", "read.logical-values", "read.statistics-metadata", "read.parquet-metadata-view"] +expected_unsupported = [] + +[[fixture]] +id = "apache-byte-array-decimal" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/byte_array_decimal.parquet" +sha256 = "9e3ccb253adc5881521b952f7b621954551df1e48dfda19e9b02126aca9b127d" +size = 324 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["semantic.logical-order", "read.logical-values"] +expected_unsupported = [] + +[[fixture]] +id = "apache-fixed-length-byte-array" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/fixed_length_byte_array.parquet" +sha256 = "a5a24cfabf2d8882db861502a0fe1e4539a80772f472f014637a0d01519836a7" +size = 4437 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.logical-order", "read.logical-values", "read.statistics-metadata", "read.parquet-metadata-view"] +expected_unsupported = [] + +[[fixture]] +id = "apache-fixed-length-decimal" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/fixed_length_decimal.parquet" +sha256 = "67e61d18ecca6027731faf397c5981b29d863f9eef68e3644501588663e1bfd2" +size = 677 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.statistics.deprecated-bounds", "wire.statistics.counts", "semantic.logical-order", "compat.legacy-statistics", "read.logical-values"] +expected_unsupported = [] + +[[fixture]] +id = "apache-fixed-length-decimal-legacy" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/fixed_length_decimal_legacy.parquet" +sha256 = "323ff9d3379903d528cbf7b00f93e4598ed71d39aa6eeff3d322fff541c1fb2a" +size = 537 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.statistics.deprecated-bounds", "wire.statistics.counts", "semantic.logical-order", "compat.legacy-statistics", "read.logical-values"] +expected_unsupported = [] + +[[fixture]] +id = "apache-float16-nonzeros-and-nans" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/float16_nonzeros_and_nans.parquet" +sha256 = "d0117dd9655992b869f8207235526a7d8931e079fd68d68c88c0170faa1f11ee" +size = 501 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.deprecated-bounds", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.type-order", "semantic.logical-order"] +expected_unsupported = [] + +[[fixture]] +id = "apache-float16-zeros-and-nans" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/float16_zeros_and_nans.parquet" +sha256 = "4901850e7dcd64588a49391fa1dddca514b598e6f4266239033ea73513850f47" +size = 489 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.deprecated-bounds", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.type-order", "semantic.logical-order"] +expected_unsupported = [] + +[[fixture]] +id = "apache-floating-orders-nan-count" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/floating_orders_nan_count.parquet" +sha256 = "17f7d7655a089b9504a828dffaccd72225a9a6fd2a697099b5336ab274386f0a" +size = 6143 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 5 +leaf_count = 6 +normalized_record_count = 31 +capabilities = ["wire.column-order.type", "wire.column-order.ieee", "wire.statistics.deprecated-bounds", "wire.statistics.modern-bounds", "wire.statistics.counts", "wire.statistics.nan-count", "semantic.type-order", "semantic.ieee-total-order", "semantic.count-state"] +expected_unsupported = [] + +[[fixture]] +id = "apache-int32-decimal" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/int32_decimal.parquet" +sha256 = "3441daea2c44032a78a3615b82373f34575ba7d820541e821f86d8cc143653f9" +size = 478 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.statistics.deprecated-bounds", "wire.statistics.counts", "semantic.logical-order", "compat.legacy-statistics", "read.logical-values"] +expected_unsupported = [] + +[[fixture]] +id = "apache-int32-with-null-pages" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/int32_with_null_pages.parquet" +sha256 = "392046fe71c7bdf7ea59e258596b5e6919f01f65f702a27ca56d8763d2e9f9b7" +size = 3829 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.deprecated-bounds", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.type-order", "semantic.count-state", "read.logical-values", "read.statistics-metadata", "read.parquet-metadata-view"] +expected_unsupported = [] + +[[fixture]] +id = "apache-int64-decimal" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/int64_decimal.parquet" +sha256 = "e24dcf95589ee230636e228ad75aa0496eca7e8f97339d9bb6ec5c8f7ab0ef56" +size = 591 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.statistics.deprecated-bounds", "wire.statistics.counts", "semantic.logical-order", "compat.legacy-statistics", "read.logical-values"] +expected_unsupported = [] + +[[fixture]] +id = "apache-int96-timestamp-order" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/int96_timestamp_order.parquet" +sha256 = "e35f8748d286a729e719a01c5411f81c79802d61a55046d5cf9a663918a14644" +size = 427 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.empty", "wire.statistics.modern-bounds", "wire.statistics.counts"] +expected_unsupported = [] + +[[fixture]] +id = "apache-json" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/json.parquet" +sha256 = "594f8dca52a6428e4350d12faeaca9e2155c77511abb9122b935bd5be1a26bf5" +size = 402 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.logical-order", "read.logical-values", "read.statistics-metadata", "read.parquet-metadata-view"] +expected_unsupported = [] + +[[fixture]] +id = "apache-nan-in-stats" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/nan_in_stats.parquet" +sha256 = "77d921ab7bed54232da778f920f423bd821075353b6147e3680f5b20c85f6337" +size = 329 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.deprecated-bounds", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.type-order", "read.statistics-metadata", "read.parquet-metadata-view"] +expected_unsupported = [] + +[[fixture]] +id = "apache-rle-boolean-encoding" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/rle_boolean_encoding.parquet" +sha256 = "585e22b54c482befc54fc6caaea5efce788f1d0737505c2d8b121da8ac0c7d76" +size = 192 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.statistics.deprecated-bounds", "wire.statistics.modern-bounds", "wire.statistics.counts", "semantic.type-order", "read.logical-values"] +expected_unsupported = [] + +[[fixture]] +id = "apache-single-nan" +status = "verified" +source_kind = "apache-corpus" +authority = "parquet-testing" +file = "data/single_nan.parquet" +sha256 = "ea3371c44ed1794843a2f529888120537f68aedcb80d6fbe32cea1003ab5769e" +size = 660 +source_revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +row_group_count = 1 +leaf_count = 1 +normalized_record_count = 2 +capabilities = ["wire.column-order.type", "wire.statistics.counts", "semantic.count-state"] +expected_unsupported = [] diff --git a/test/conformance/n6/generated/julia-reader-no-pruning-absent.parquet b/test/conformance/n6/generated/julia-reader-no-pruning-absent.parquet new file mode 100644 index 0000000..5c5d8b0 Binary files /dev/null and b/test/conformance/n6/generated/julia-reader-no-pruning-absent.parquet differ diff --git a/test/conformance/n6/generated/julia-reader-no-pruning-invalid.parquet b/test/conformance/n6/generated/julia-reader-no-pruning-invalid.parquet new file mode 100644 index 0000000..253f024 Binary files /dev/null and b/test/conformance/n6/generated/julia-reader-no-pruning-invalid.parquet differ diff --git a/test/conformance/n6/generated/julia-reader-no-pruning-oversized.parquet b/test/conformance/n6/generated/julia-reader-no-pruning-oversized.parquet new file mode 100644 index 0000000..0fe4aa5 Binary files /dev/null and b/test/conformance/n6/generated/julia-reader-no-pruning-oversized.parquet differ diff --git a/test/conformance/n6/generated/julia-reader-no-pruning-trusted.parquet b/test/conformance/n6/generated/julia-reader-no-pruning-trusted.parquet new file mode 100644 index 0000000..22f41a6 Binary files /dev/null and b/test/conformance/n6/generated/julia-reader-no-pruning-trusted.parquet differ diff --git a/test/conformance/n6/generated/julia-reader-no-pruning-untrusted.parquet b/test/conformance/n6/generated/julia-reader-no-pruning-untrusted.parquet new file mode 100644 index 0000000..18582c5 Binary files /dev/null and b/test/conformance/n6/generated/julia-reader-no-pruning-untrusted.parquet differ diff --git a/test/conformance/n6/generated/julia-reader-untrusted-producer.parquet b/test/conformance/n6/generated/julia-reader-untrusted-producer.parquet new file mode 100644 index 0000000..d416090 Binary files /dev/null and b/test/conformance/n6/generated/julia-reader-untrusted-producer.parquet differ diff --git a/test/conformance/n6/generated/julia-writer-ieee-order.parquet b/test/conformance/n6/generated/julia-writer-ieee-order.parquet new file mode 100644 index 0000000..5576c04 Binary files /dev/null and b/test/conformance/n6/generated/julia-writer-ieee-order.parquet differ diff --git a/test/conformance/n6/generated/julia-writer-nested-row-groups.parquet b/test/conformance/n6/generated/julia-writer-nested-row-groups.parquet new file mode 100644 index 0000000..1201a79 Binary files /dev/null and b/test/conformance/n6/generated/julia-writer-nested-row-groups.parquet differ diff --git a/test/conformance/n6/generated/julia-writer-oversized-bounds.parquet b/test/conformance/n6/generated/julia-writer-oversized-bounds.parquet new file mode 100644 index 0000000..99fa8d9 Binary files /dev/null and b/test/conformance/n6/generated/julia-writer-oversized-bounds.parquet differ diff --git a/test/conformance/n6/generated/julia-writer-statistics-disabled.parquet b/test/conformance/n6/generated/julia-writer-statistics-disabled.parquet new file mode 100644 index 0000000..5de5d58 Binary files /dev/null and b/test/conformance/n6/generated/julia-writer-statistics-disabled.parquet differ diff --git a/test/conformance/n6/generated/julia-writer-type-order.parquet b/test/conformance/n6/generated/julia-writer-type-order.parquet new file mode 100644 index 0000000..7597ed1 Binary files /dev/null and b/test/conformance/n6/generated/julia-writer-type-order.parquet differ diff --git a/test/conformance/n6/generated/julia-writer-undefined-order.parquet b/test/conformance/n6/generated/julia-writer-undefined-order.parquet new file mode 100644 index 0000000..b4abadb Binary files /dev/null and b/test/conformance/n6/generated/julia-writer-undefined-order.parquet differ diff --git a/test/conformance/n6/harnesses/common.py b/test/conformance/n6/harnesses/common.py new file mode 100644 index 0000000..c601dc7 --- /dev/null +++ b/test/conformance/n6/harnesses/common.py @@ -0,0 +1,1175 @@ +#!/usr/bin/env python3 +import datetime +import decimal +import hashlib +import importlib +import io +import json +import os +import pathlib +import platform +import re +import stat +import struct +import sys +import tempfile +import tomllib +import zipfile +from contextlib import contextmanager + + +class HarnessError(Exception): + pass + + +_CONTROL_FILE_LIMIT = 4 * 1024 * 1024 +_READ_CHUNK = 1024 * 1024 +_WHEEL_FILE_LIMIT = 128 * 1024 * 1024 +_WHEEL_MEMBER_LIMIT = 256 * 1024 * 1024 +_WHEEL_TOTAL_LIMIT = 512 * 1024 * 1024 +_WHEEL_MEMBER_COUNT_LIMIT = 4096 +_AUTHENTICATED_SNAPSHOT_ENV = "PARQUET_N6_AUTHENTICATED_SNAPSHOT" + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def _metadata_identity(metadata): + return (metadata.st_dev, metadata.st_ino, metadata.st_mode, + metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, + metadata.st_ctime_ns) + + +@contextmanager +def stable_regular_file(path, maximum_bytes, label): + path = pathlib.Path(path) + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise HarnessError(f"{label} is not a regular file") + if not 0 <= metadata.st_size <= maximum_bytes: + raise HarnessError(f"{label} has an invalid size") + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as error: + raise HarnessError(f"cannot open {label} safely") from error + stream = os.fdopen(descriptor, "rb") + try: + opened = os.fstat(stream.fileno()) + if _metadata_identity(opened) != _metadata_identity(metadata): + raise HarnessError(f"{label} changed while it was opened") + yield stream, opened + final = os.fstat(stream.fileno()) + if _metadata_identity(final) != _metadata_identity(opened): + raise HarnessError(f"{label} changed while it was read") + finally: + stream.close() + + +def regular_file_bytes(path, maximum_bytes, label): + with stable_regular_file(path, maximum_bytes, label) as (stream, metadata): + value = stream.read(maximum_bytes + 1) + if len(value) > maximum_bytes or len(value) != metadata.st_size: + raise HarnessError(f"{label} changed size while it was read") + if stream.read(1): + raise HarnessError(f"{label} exceeds its byte limit") + return value + + +def sha256_file(path, maximum_bytes=None): + path = pathlib.Path(path) + metadata = path.lstat() + limit = metadata.st_size if maximum_bytes is None else maximum_bytes + digest = hashlib.sha256() + with stable_regular_file(path, limit, "hashed input") as (stream, opened): + total = 0 + while chunk := stream.read(_READ_CHUNK): + total += len(chunk) + if total > limit: + raise HarnessError("hashed input exceeds its byte limit") + digest.update(chunk) + if total != opened.st_size: + raise HarnessError("hashed input changed size while it was read") + return digest.hexdigest() + + +def canonical_json(value, *, ascii_only=False): + return json.dumps(value, allow_nan=False, ensure_ascii=ascii_only, + separators=(",", ":"), sort_keys=True) + + +def observation_digest(case_id, capability_id, observations): + envelope = { + "capability_id": capability_id, + "case_id": case_id, + "observations": observations, + } + return sha256_bytes(canonical_json(envelope).encode("utf-8")) + + +def safe_relative(value): + if not value or "\\" in value or value.startswith("/"): + return False + parts = value.split("/") + if any(part in ("", ".", "..") for part in parts): + return False + return all(character.isascii() and (character.isalnum() or + character in "_ .+@=-/".replace(" ", "")) for character in value) + + +def checked_file(root, relative, expected_sha256, expected_size, + snapshot_directory): + if not safe_relative(relative): + raise HarnessError(f"unsafe fixture path: {relative!r}") + root_input = pathlib.Path(root) + if root_input.is_symlink(): + raise HarnessError("fixture root is a symbolic link") + root = root_input.resolve(strict=True) + candidate = root.joinpath(*pathlib.PurePosixPath(relative).parts) + current = root + for part in pathlib.PurePosixPath(relative).parts: + current = current / part + if current.is_symlink(): + raise HarnessError(f"fixture path contains a symbolic link: {relative}") + metadata = candidate.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise HarnessError(f"fixture is not a regular file: {relative}") + resolved = candidate.resolve(strict=True) + try: + resolved.relative_to(root) + except ValueError as error: + raise HarnessError(f"fixture escapes its root: {relative}") from error + value = regular_file_bytes(resolved, expected_size, + f"fixture {relative}") + if len(value) != expected_size: + raise HarnessError(f"fixture size differs: {relative}") + if sha256_bytes(value) != expected_sha256: + raise HarnessError(f"fixture digest differs: {relative}") + snapshot_root = pathlib.Path(snapshot_directory) + snapshot_metadata = snapshot_root.lstat() + if stat.S_ISLNK(snapshot_metadata.st_mode) or \ + not stat.S_ISDIR(snapshot_metadata.st_mode): + raise HarnessError("fixture snapshot root is not a directory") + descriptor, snapshot = tempfile.mkstemp(prefix="fixture-", suffix=".parquet", + dir=snapshot_root) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.unlink(snapshot) + except FileNotFoundError: + pass + raise + return pathlib.Path(snapshot) + + +def load_toml_snapshot(path, label="TOML input"): + source = regular_file_bytes(path, _CONTROL_FILE_LIMIT, label) + try: + value = tomllib.loads(source.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise HarnessError(f"{label} is invalid") from error + return value, source, sha256_bytes(source) + + +def load_toml(path): + value, _, _ = load_toml_snapshot(path) + return value + + +def tree_sha256(root): + root_input = pathlib.Path(root) + if root_input.is_symlink(): + raise HarnessError("Python runtime root is a symbolic link") + root = root_input.resolve(strict=True) + entries = [] + for directory, directories, files in os.walk(root, followlinks=False): + for name in directories + files: + path = pathlib.Path(directory) / name + if path.is_file() or path.is_symlink(): + entries.append(path.relative_to(root).as_posix()) + digest = hashlib.sha256() + for relative in sorted(entries): + path = root.joinpath(*pathlib.PurePosixPath(relative).parts) + if path.is_symlink(): + row = b"L\0" + relative.encode("utf-8") + b"\0" + \ + os.readlink(path).encode("utf-8") + b"\n" + else: + row = b"F\0" + relative.encode("utf-8") + b"\0" + \ + sha256_file(path).encode("ascii") + b"\n" + digest.update(row) + return digest.hexdigest() + + +def runtime_tree_policy_violations(root): + root = pathlib.Path(root).resolve(strict=True) + violations = [] + for directory, directories, files in os.walk(root, followlinks=False): + for name in directories: + if name in ("site-packages", "__pycache__"): + path = pathlib.Path(directory, name).relative_to(root) + violations.append(path.as_posix()) + for name in files: + if name.endswith(".pyc"): + path = pathlib.Path(directory, name).relative_to(root) + violations.append(path.as_posix()) + if len(violations) >= 16: + break + return sorted(violations) + + +def _wheel_entry_path(info): + name = info.filename + relative = name[:-1] if name.endswith("/") else name + if not safe_relative(relative): + raise HarnessError(f"wheel member path is unsafe: {name!r}") + parts = pathlib.PurePosixPath(relative).parts + if parts[0].endswith(".data"): + raise HarnessError("wheel .data installation schemes are unsupported") + mode = info.external_attr >> 16 + kind = stat.S_IFMT(mode) + expected_kinds = (0, stat.S_IFDIR) if info.is_dir() else (0, stat.S_IFREG) + if kind not in expected_kinds: + raise HarnessError(f"wheel member is not a regular file: {name}") + if info.flag_bits & 1: + raise HarnessError(f"wheel member is encrypted: {name}") + if info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): + raise HarnessError(f"wheel member compression is unsupported: {name}") + if not 0 <= info.file_size <= _WHEEL_MEMBER_LIMIT: + raise HarnessError(f"wheel member exceeds its size limit: {name}") + return parts + + +def _wheel_descriptor_entry(descriptor): + wheels = descriptor.get("wheels") + if not isinstance(wheels, list) or len(wheels) != 1: + raise HarnessError("Python descriptor must name exactly one wheel") + entry = wheels[0] + if not isinstance(entry, dict) or set(entry) != {"name", "sha256"} or \ + not isinstance(entry["name"], str) or \ + not isinstance(entry["sha256"], str) or \ + pathlib.PurePosixPath(entry["name"]).name != entry["name"] or \ + not safe_relative(entry["name"]) or len(entry["sha256"]) != 64 or \ + any(character not in "0123456789abcdef" + for character in entry["sha256"]): + raise HarnessError("Python descriptor wheel entry is invalid") + return entry + + +@contextmanager +def wheel_import_root(wheel_path, descriptor): + entry = _wheel_descriptor_entry(descriptor) + wheel = pathlib.Path(wheel_path) + if wheel.name != entry["name"]: + raise HarnessError("wheel filename differs from its descriptor") + value = regular_file_bytes(wheel, _WHEEL_FILE_LIMIT, "wheel input") + if sha256_bytes(value) != entry["sha256"]: + raise HarnessError("wheel digest differs from its descriptor") + try: + archive = zipfile.ZipFile(io.BytesIO(value)) + except zipfile.BadZipFile as error: + raise HarnessError("wheel input is not a valid ZIP archive") from error + with archive: + infos = archive.infolist() + if not 0 < len(infos) <= _WHEEL_MEMBER_COUNT_LIMIT: + raise HarnessError("wheel member count exceeds its limit") + names = [info.filename for info in infos] + if len(names) != len(set(names)): + raise HarnessError("wheel contains duplicate member names") + paths = [_wheel_entry_path(info) for info in infos] + if sum(info.file_size for info in infos) > _WHEEL_TOTAL_LIMIT: + raise HarnessError("wheel expands beyond its total size limit") + dist_info = {parts[0] for parts in paths + if parts[0].endswith(".dist-info")} + if len(dist_info) != 1: + raise HarnessError("wheel dist-info directory is ambiguous") + required = { + next(iter(dist_info)) + "/RECORD", + next(iter(dist_info)) + "/WHEEL", + } + if not required.issubset(names): + raise HarnessError("wheel metadata files are incomplete") + with tempfile.TemporaryDirectory(prefix="parquet-n6-wheel-") as directory: + root = pathlib.Path(directory).resolve(strict=True) + for info, parts in zip(infos, paths): + destination = root.joinpath(*parts) + if info.is_dir(): + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + continue + destination.parent.mkdir(parents=True, exist_ok=True, + mode=0o700) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor_fd = os.open(destination, flags, 0o600) + total = 0 + try: + with os.fdopen(descriptor_fd, "wb") as output, \ + archive.open(info, "r") as source: + while chunk := source.read(_READ_CHUNK): + total += len(chunk) + if total > info.file_size: + raise HarnessError( + f"wheel member changed size: {info.filename}") + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + except BaseException: + try: + destination.unlink() + except FileNotFoundError: + pass + raise + if total != info.file_size: + raise HarnessError( + f"wheel member changed size: {info.filename}") + yield root + + +def _wheel_owned_modules(root): + owned = set() + for path in pathlib.Path(root).iterdir(): + if path.is_dir(): + if path.name.isidentifier(): + owned.add(path.name) + continue + if path.is_file() and path.name.endswith((".py", ".pyc", ".so", ".pyd")): + name = path.name.split(".", 1)[0] + if name.isidentifier(): + owned.add(name) + return owned + + +def _owned_module_name(name, owned): + return any(name == top or name.startswith(top + ".") for top in owned) + + +def _verify_module_origin(name, module, root): + origins = [] + module_file = getattr(module, "__file__", None) + if module_file is not None: + origins.append(module_file) + specification = getattr(module, "__spec__", None) + specification_origin = getattr(specification, "origin", None) + if specification_origin is not None and specification_origin not in origins: + origins.append(specification_origin) + locations = getattr(specification, "submodule_search_locations", None) + if not origins and locations is not None: + origins.extend(locations) + if not origins: + raise HarnessError(f"wheel module has no file origin: {name}") + for source in origins: + if not isinstance(source, (str, os.PathLike)) or source in ( + "built-in", "frozen"): + raise HarnessError(f"wheel module has an invalid origin: {name}") + try: + pathlib.Path(source).resolve(strict=True).relative_to(root) + except (OSError, ValueError) as error: + raise HarnessError( + f"module did not load from the pinned wheel: {name}") from error + return + + +def _verify_owned_module_origins(owned, root): + loaded = {name: module for name, module in sys.modules.items() + if _owned_module_name(name, owned)} + for name, module in loaded.items(): + _verify_module_origin(name, module, root) + return + + +@contextmanager +def exact_wheel_module(wheel_path, descriptor, module_name, version): + with wheel_import_root(wheel_path, descriptor) as root: + owned = _wheel_owned_modules(root) + if module_name not in owned: + raise HarnessError(f"wheel does not own its requested module: {module_name}") + preloaded = sorted(name for name in sys.modules + if _owned_module_name(name, owned)) + if preloaded: + raise HarnessError( + f"wheel module is already loaded: {preloaded[0]}") + sys.path.insert(0, str(root)) + try: + module = importlib.import_module(module_name) + _verify_owned_module_origins(owned, root) + if getattr(module, "__version__", None) != version: + raise HarnessError(f"wheel module version differs: {module_name}") + yield module + finally: + try: + _verify_owned_module_origins(owned, root) + finally: + try: + sys.path.remove(str(root)) + except ValueError: + pass + for name in [name for name in sys.modules + if _owned_module_name(name, owned)]: + del sys.modules[name] + importlib.invalidate_caches() + + +def verify_python_runtime(descriptor): + if sys.implementation.name != "cpython": + raise HarnessError("Python runtime is not CPython") + expected_version = tuple(int(part) for part in + descriptor["python_version"].split(".")) + if len(expected_version) != 3 or sys.version_info[:3] != expected_version: + raise HarnessError("Python runtime version differs") + if not sys.dont_write_bytecode: + raise HarnessError("Python runtime must use -B") + if not sys.flags.isolated: + raise HarnessError("Python runtime must use -I") + if not sys.flags.no_site: + raise HarnessError("Python runtime must use -S") + distribution_url = descriptor.get("python_distribution_url") + if not isinstance(distribution_url, str) or not distribution_url.startswith( + "https://github.com/astral-sh/python-build-standalone/" + "releases/download/") or not distribution_url.endswith(".tar.gz") or \ + any(character.isspace() for character in distribution_url): + raise HarnessError("Python distribution URL is invalid") + distribution_sha256 = descriptor.get("python_distribution_sha256") + if not isinstance(distribution_sha256, str) or re.fullmatch( + r"[0-9a-f]{64}", distribution_sha256) is None: + raise HarnessError("Python distribution digest is invalid") + if descriptor.get("python_tree_policy") != \ + "extract-strip-site-packages-bytecode-v1": + raise HarnessError("Python runtime tree policy differs") + violations = runtime_tree_policy_violations(sys.base_prefix) + if violations: + raise HarnessError( + f"Python runtime tree violates its clean policy: {violations[0]}") + executable = pathlib.Path(sys.executable).resolve(strict=True) + if sha256_file(executable) != descriptor["python_executable_sha256"]: + raise HarnessError("Python executable digest differs") + runtime_root = pathlib.Path(sys.base_prefix) + if tree_sha256(runtime_root) != descriptor["python_tree_sha256"]: + raise HarnessError("Python runtime tree digest differs") + if any("site-packages" in pathlib.PurePath(entry).parts for entry in sys.path): + raise HarnessError("Python runtime loaded a site-packages path") + system = platform.system() + machine = platform.machine() + macos = platform.mac_ver()[0].split(".", 1)[0] + actual_platform = f"macos-{macos}-{machine}" if system == "Darwin" else \ + f"{system.lower()}-{machine}" + if actual_platform != descriptor["platform"]: + raise HarnessError("Python runtime platform differs") + return + + +def authority(capabilities, producer): + matches = [item for item in capabilities["authority"] + if item["id"] == producer] + if len(matches) != 1: + raise HarnessError(f"authority is absent or ambiguous: {producer}") + return matches[0] + + +def fixture_map(fixtures): + output = {} + for source in fixtures["fixture"]: + item = dict(source) + item["digest_contract"] = fixtures["default_digest_contract"] + item["generated"] = False + item["semantic"] = False + if item["id"] in output: + raise HarnessError(f"duplicate fixture ID: {item['id']}") + output[item["id"]] = item + for source in fixtures["generated_case"]: + item = dict(source) + item["file"] = item["output_file"] + item["generated"] = True + item["semantic"] = False + if item["id"] in output: + raise HarnessError(f"duplicate fixture ID: {item['id']}") + output[item["id"]] = item + return output + + +def semantic_case_map(repository, manifest, capabilities, fixtures): + relative = "test/conformance/n6/model/cases.toml" + entries = [item for item in manifest["frozen_model"] + if item["file"] == relative] + if len(entries) != 1: + raise HarnessError("semantic case manifest is absent or ambiguous") + path = repository_input(repository, relative) + value, _, digest = load_toml_snapshot(path, "semantic case manifest") + if digest != entries[0]["sha256"]: + raise HarnessError("semantic case manifest digest differs") + if value.get("schema_version") != 1 or \ + not isinstance(value.get("case_groups"), list): + raise HarnessError("semantic case manifest header differs") + capability_ids = {item["id"] for item in capabilities["capability"]} + contracts = set(fixtures["digest_contracts"]) + output = {} + for source in value["case_groups"]: + required = {"id", "requirements", "capabilities", "digest_contract", + "expected_sha256"} + if not isinstance(source, dict) or set(source) != required: + raise HarnessError("semantic case fields differ") + identifier = source["id"] + case_capabilities = source["capabilities"] + expected = source["expected_sha256"] + if not isinstance(identifier, str) or re.fullmatch( + r"[a-z0-9]+(?:[._-][a-z0-9]+)*", identifier) is None or \ + identifier in output: + raise HarnessError("semantic case ID is invalid or duplicated") + if not isinstance(source["requirements"], list) or \ + not source["requirements"] or any(not isinstance(item, str) or + not item for item in source["requirements"]): + raise HarnessError(f"semantic requirements differ: {identifier}") + if not isinstance(case_capabilities, list) or \ + case_capabilities != sorted(set(case_capabilities)) or \ + any(item not in capability_ids for item in case_capabilities): + raise HarnessError(f"semantic capabilities differ: {identifier}") + if source["digest_contract"] not in contracts or \ + not isinstance(expected, dict) or \ + set(expected) != set(case_capabilities) or \ + any(not isinstance(digest, str) or + re.fullmatch(r"[0-9a-f]{64}", digest) is None + for digest in expected.values()): + raise HarnessError(f"semantic digests differ: {identifier}") + item = dict(source) + item["generated"] = False + item["semantic"] = True + output[identifier] = item + return output, path + + +def selected_claims(authority_record): + output = {} + for claim in authority_record["claim"]: + if claim["status"] == "not_assessed": + continue + if claim["status"] not in ("planned", "verified", "unsupported"): + raise HarnessError(f"invalid claim status: {claim['status']}") + for case_id in claim["cases"]: + key = (case_id, claim["capability"]) + if key in output: + raise HarnessError(f"duplicate authority claim: {key}") + output[key] = claim["status"] + return output + + +def load_raw_facts(path, maximum_bytes, maximum_line_bytes, maximum_records): + path = pathlib.Path(path) + files = {} + columns = {} + run = None + digest = hashlib.sha256() + + def unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise HarnessError(f"duplicate raw evidence key: {key}") + value[key] = item + return value + + def invalid_number(value): + raise HarnessError(f"invalid JSON number: {value}") + + def canonical_integer(value): + if value != "0" and (value.startswith("0") or value.startswith("-0")): + raise HarnessError(f"noncanonical JSON integer: {value}") + return int(value) + + with stable_regular_file(path, maximum_bytes, + "normalized raw evidence") as (stream, metadata): + if metadata.st_size == 0: + raise HarnessError("normalized raw evidence is empty") + line_number = 0 + total = 0 + while True: + line = stream.readline(maximum_line_bytes + 1) + if not line: + break + line_number += 1 + if line_number > maximum_records: + raise HarnessError("normalized raw evidence has too many records") + if len(line) > maximum_line_bytes: + raise HarnessError( + f"normalized raw evidence line {line_number} is too large") + if not line.endswith(b"\n"): + raise HarnessError( + f"normalized raw evidence line {line_number} has no LF") + total += len(line) + if total > maximum_bytes: + raise HarnessError("normalized raw evidence exceeds its byte limit") + digest.update(line) + try: + record = json.loads(line, object_pairs_hook=unique_object, + parse_constant=invalid_number, parse_float=invalid_number, + parse_int=canonical_integer) + except (json.JSONDecodeError, UnicodeError) as error: + raise HarnessError( + f"raw evidence line {line_number} is invalid JSON") from error + if not isinstance(record, dict): + raise HarnessError( + f"raw evidence line {line_number} is not an object") + expected = canonical_json(record, ascii_only=True).encode("utf-8") + \ + b"\n" + if line != expected: + raise HarnessError( + f"raw evidence line {line_number} is not canonical JSONL") + kind = record.get("record") + if kind == "run": + if line_number != 1 or run is not None: + raise HarnessError("raw run record is absent or misplaced") + run = record + elif kind == "file": + case_id = record["case_id"] + if case_id in files: + raise HarnessError(f"duplicate raw file fact: {case_id}") + files[case_id] = record + elif kind == "column_statistics": + key = (record["case_id"], record["row_group"], record["leaf"]) + if key in columns: + raise HarnessError(f"duplicate raw column fact: {key}") + columns[key] = record + elif kind != "case_result": + raise HarnessError( + f"unknown raw evidence record on line {line_number}") + if total != metadata.st_size: + raise HarnessError( + "normalized raw evidence changed size while it was read") + if run is None or run.get("producer") != "n6-raw-java" or \ + run.get("schema_version") != 2: + raise HarnessError("normalized raw evidence has the wrong run record") + return run, files, columns, digest.hexdigest() + + +def leaf_records(case_id, raw_columns): + selected = [record for (current, row_group, _), record in raw_columns.items() + if current == case_id and row_group == 0] + selected.sort(key=lambda record: record["leaf"]) + if [record["leaf"] for record in selected] != list(range(len(selected))): + raise HarnessError(f"raw leaf ordinals are incomplete: {case_id}") + return selected + + +def epoch_nanoseconds(value): + if value.tzinfo is None: + epoch = datetime.datetime(1970, 1, 1) + else: + value = value.astimezone(datetime.timezone.utc) + epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + delta = value - epoch + return ((delta.days * 86400 + delta.seconds) * 1_000_000_000 + + delta.microseconds * 1000) + + +def decimal_unscaled(value, scale): + if not value.is_finite(): + raise HarnessError("DECIMAL value is not finite") + parts = value.as_tuple() + coefficient = 0 + for digit in parts.digits: + coefficient = coefficient * 10 + digit + shift = parts.exponent + scale + if shift >= 0: + coefficient *= 10 ** shift + else: + divisor = 10 ** -shift + coefficient, remainder = divmod(coefficient, divisor) + if remainder != 0: + raise HarnessError("DECIMAL value does not fit its declared scale") + if parts.sign: + coefficient = -coefficient + return str(coefficient) + + +def canonical_value(value, leaf_schema): + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, bytes): + return {"bytes_hex": value.hex()} + if isinstance(value, str): + return value + if isinstance(value, decimal.Decimal): + scale = leaf_schema["scale"] + if scale is None: + raise HarnessError("DECIMAL value has no declared scale") + return {"decimal_scale": scale, + "unscaled": decimal_unscaled(value, scale)} + if isinstance(value, datetime.datetime): + return {"timestamp_nanoseconds": str(epoch_nanoseconds(value))} + if isinstance(value, datetime.date): + return {"date_days": (value - datetime.date(1970, 1, 1)).days} + if isinstance(value, datetime.time): + nanoseconds = ((value.hour * 3600 + value.minute * 60 + value.second) * + 1_000_000_000 + value.microsecond * 1000) + return {"time_nanoseconds": str(nanoseconds)} + if isinstance(value, int): + return value + if isinstance(value, float): + physical = leaf_schema["physical_type"] + if physical == "FLOAT": + return {"float32_bits": struct.pack(">f", value).hex()} + if physical == "DOUBLE": + return {"float64_bits": struct.pack(">d", value).hex()} + raise HarnessError(f"floating value has physical type {physical}") + raise HarnessError(f"unsupported logical value type: {type(value).__name__}") + + +def logical_observations(case_id, raw_columns, columns, row_count): + leaves = leaf_records(case_id, raw_columns) + if len(columns) != len(leaves): + raise HarnessError(f"decoded leaf count differs: {case_id}") + normalized = [] + for leaf, values in zip(leaves, columns): + if len(values) != row_count: + raise HarnessError(f"decoded column length differs: {case_id}") + normalized.append({ + "logical_type": leaf["leaf_schema"]["logical_type"], + "path": leaf["path"], + "physical_type": leaf["leaf_schema"]["physical_type"], + "values": [canonical_value(value, leaf["leaf_schema"]) + for value in values], + }) + return [{ + "columns": normalized, + "contract": "n6-logical-values-v1", + "row_count": row_count, + }] + + +def frozen_input_paths(root, manifest): + paths = { + "plan_sha256": manifest["plan_file"], + "capabilities_sha256": manifest["capabilities_file"], + "fixture_manifest_sha256": manifest["fixture_manifest_file"], + "corpus_manifest_sha256": manifest["corpus_manifest_file"], + "evidence_schema_sha256": manifest["evidence_schema_file"], + } + return {field: repository_input(root, relative) + for field, relative in paths.items()} + + +def frozen_input_hashes(root, manifest, known_hashes=None): + known_hashes = {} if known_hashes is None else known_hashes + return {field: known_hashes[field] if field in known_hashes else + sha256_file(path) for field, path in + frozen_input_paths(root, manifest).items()} + + +def run_record(evidence_id, producer, authority_record, descriptor_sha256, + root, manifest, unsupported_cases, upstream_evidence, + input_hashes=None): + record = { + "record": "run", + "schema_version": 2, + "evidence_id": evidence_id, + "producer": producer, + "producer_version": authority_record["version"], + "source_revision": authority_record["revision"], + "toolchain_sha256": descriptor_sha256, + "unsupported_cases": sorted(unsupported_cases), + "upstream_evidence": [upstream_evidence], + } + hashes = frozen_input_hashes(root, manifest) if input_hashes is None else \ + input_hashes + record.update(hashes) + return record + + +def case_result(case_id, capability_id, contract, status, observations=None, + expected_sha256=None): + if status == "UNSUPPORTED": + return { + "record": "case_result", + "schema_version": 2, + "case_id": case_id, + "capability_id": capability_id, + "digest_contract": contract, + "status": status, + "expected_sha256": None, + "actual_sha256": None, + "detail": "The reviewed capability matrix marks this result unsupported.", + } + if status != "PASS": + raise HarnessError(f"unsupported requested result status: {status}") + actual = observation_digest(case_id, capability_id, observations) + expected = actual if expected_sha256 is None else expected_sha256 + if not isinstance(expected, str) or re.fullmatch( + r"[0-9a-f]{64}", expected) is None: + raise HarnessError("expected capability digest is invalid") + result_status = "PASS" if actual == expected else "FAIL" + return { + "record": "case_result", + "schema_version": 2, + "case_id": case_id, + "capability_id": capability_id, + "digest_contract": contract, + "status": result_status, + "expected_sha256": expected, + "actual_sha256": actual, + "detail": "Observed values match the frozen canonical digest." + if result_status == "PASS" else + "Observed values differ from the frozen canonical digest.", + } + + +def evidence_bytes(records, maximum_bytes, maximum_line_bytes, maximum_records): + if not 0 < len(records) <= maximum_records: + raise HarnessError("evidence record count exceeds its limit") + output = bytearray() + for record in records: + line = canonical_json(record, ascii_only=True).encode("utf-8") + b"\n" + if len(line) > maximum_line_bytes: + raise HarnessError("evidence line exceeds its byte limit") + output.extend(line) + if len(output) > maximum_bytes: + raise HarnessError("evidence output exceeds its byte limit") + return bytes(output) + + +def output_path(path, create_parents): + requested = pathlib.Path(os.path.abspath(path)) + if create_parents: + requested.parent.mkdir(parents=True, exist_ok=True) + elif not requested.parent.is_dir(): + raise HarnessError("evidence output parent does not exist") + parent = requested.parent.resolve(strict=True) + if not parent.is_dir(): + raise HarnessError("evidence output parent is not a directory") + path = parent / requested.name + try: + metadata = path.lstat() + except FileNotFoundError: + return path + if stat.S_ISLNK(metadata.st_mode): + raise HarnessError("evidence output is a symbolic link") + if not stat.S_ISREG(metadata.st_mode): + raise HarnessError("evidence output is not a regular file") + return path + + +def manifest_output_path(repository, target_entry, requested): + relative = target_entry.get("file") + if not isinstance(relative, str) or not safe_relative(relative): + raise HarnessError("target evidence file is invalid") + root = pathlib.Path(repository).resolve(strict=True) + expected = root.joinpath(*pathlib.PurePosixPath(relative).parts) + current = root + for part in pathlib.PurePosixPath(relative).parts[:-1]: + current = current / part + metadata = current.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise HarnessError("target evidence parent is not a safe directory") + destination = pathlib.Path(os.path.abspath(requested)) + if destination != expected: + raise HarnessError("evidence output does not select the manifest target") + return expected + + +def atomic_output(path, value, check): + path = output_path(path, not check) + if check: + try: + actual = regular_file_bytes(path, len(value), "evidence output") + except (FileNotFoundError, HarnessError) as error: + raise HarnessError("evidence output is stale") from error + if actual != value: + raise HarnessError("evidence output is stale") + return + descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", + dir=path.parent) + try: + os.fchmod(descriptor, 0o644) + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def repository_input(repository, relative): + if not safe_relative(relative): + raise HarnessError(f"unsafe repository input path: {relative!r}") + root = pathlib.Path(repository).resolve(strict=True) + candidate = root.joinpath(*pathlib.PurePosixPath(relative).parts) + current = root + for part in pathlib.PurePosixPath(relative).parts: + current = current / part + if current.is_symlink(): + raise HarnessError( + f"repository input contains a symbolic link: {relative}") + if not candidate.is_file(): + raise HarnessError(f"repository input is not a regular file: {relative}") + return candidate + + +def evidence_entry(manifest, evidence_id): + matches = [] + for section in ("frozen_evidence", "planned_evidence"): + matches.extend(item for item in manifest.get(section, []) + if item["id"] == evidence_id) + if len(matches) != 1: + raise HarnessError( + f"evidence entry is absent or ambiguous: {evidence_id}") + return matches[0] + + +def authenticated_source_overrides(repository, descriptor, descriptor_path, + executed_harness): + snapshot_value = os.environ.get(_AUTHENTICATED_SNAPSHOT_ENV) + if snapshot_value is None: + raise HarnessError("authenticated source snapshot is absent") + snapshot = pathlib.Path(snapshot_value).resolve(strict=True) + descriptor_path = pathlib.Path(descriptor_path).resolve(strict=True) + harness_path = pathlib.Path(executed_harness).resolve(strict=True) + common_path = pathlib.Path(__file__).resolve(strict=True) + if descriptor_path != snapshot / "descriptor.toml" or \ + harness_path.parent != snapshot or common_path != snapshot / \ + "common.py": + raise HarnessError("authenticated source snapshot paths differ") + harness_relative = descriptor.get("harness_file") + if not isinstance(harness_relative, str) or harness_path.name != \ + pathlib.PurePosixPath(harness_relative).name: + raise HarnessError("authenticated harness source differs") + common_relative = (pathlib.PurePosixPath(harness_relative).parent / + "common.py").as_posix() + matches = [item for item in descriptor.get("support_files", []) + if isinstance(item, dict) and item.get("file") == common_relative] + if len(matches) != 1: + raise HarnessError("authenticated common source is not declared") + repository_input(repository, descriptor["test_file"]) + return { + harness_relative: harness_path, + common_relative: common_path, + } + + +def descriptor_repository_inputs(repository, descriptor, + source_overrides=None): + source_overrides = {} if source_overrides is None else source_overrides + declared = [] + if "harness_file" in descriptor: + declared.append((descriptor["harness_file"], + descriptor.get("harness_sha256"))) + if "test_file" in descriptor: + declared.append((descriptor["test_file"], + descriptor.get("test_sha256"))) + for item in descriptor.get("support_files", []): + declared.append((item.get("file"), item.get("sha256"))) + for item in descriptor.get("wrapper", []): + declared.append((item.get("path"), item.get("sha256"))) + paths = [] + for relative, expected in declared: + if not isinstance(relative, str) or not isinstance(expected, str): + raise HarnessError("descriptor source binding is incomplete") + path = source_overrides.get(relative) + if path is None: + path = repository_input(repository, relative) + else: + path = pathlib.Path(path).resolve(strict=True) + if sha256_file(path) != expected: + raise HarnessError(f"descriptor source digest differs: {relative}") + paths.append(path) + return paths + + +def verify_python_descriptor(descriptor, repository, producer, + source_overrides=None, descriptor_inputs=None): + if descriptor.get("authority") != producer: + raise HarnessError("Python descriptor authority differs") + if descriptor.get("harness_status") != descriptor.get("status"): + raise HarnessError("Python descriptor harness status differs") + required = ("python_version", "python_executable_sha256", + "python_tree_sha256", "python_distribution_url", + "python_distribution_sha256", "python_tree_policy", "platform", + "harness_file", "harness_sha256", "test_file", "test_sha256", + "support_files", "wheels") + if any(field not in descriptor for field in required): + raise HarnessError("Python descriptor is incomplete") + if descriptor_inputs is None: + descriptor_repository_inputs(repository, descriptor, source_overrides) + _wheel_descriptor_entry(descriptor) + verify_python_runtime(descriptor) + return + + +def reject_output_alias(output, inputs, protected_roots): + destination = pathlib.Path(os.path.abspath(output)).resolve(strict=False) + for source in inputs: + resolved_source = pathlib.Path(source).resolve(strict=True) + same_file = destination.exists() and os.path.samefile(destination, + resolved_source) + if destination == resolved_source or same_file: + raise HarnessError("evidence output aliases an input") + for source in protected_roots: + root = pathlib.Path(source).resolve(strict=True) + try: + destination.relative_to(root) + except ValueError: + continue + raise HarnessError("evidence output is inside a protected input root") + + +def _verify_evidence_bindings(manifest, producer, evidence_id, raw_entry, + raw_evidence_sha256, draft): + raw_id = "normalized-raw-java-apache-corpus" + if raw_entry.get("authority") != "n6-raw-java" or \ + raw_entry.get("format") != "normalized-jsonl": + raise HarnessError("normalized raw evidence declaration differs") + raw_status = raw_entry.get("status") + if raw_status == "verified": + if raw_entry.get("sha256") != raw_evidence_sha256: + raise HarnessError("normalized raw evidence digest differs") + elif not draft or raw_status != "planned" or "sha256" in raw_entry: + raise HarnessError("normalized raw evidence is not frozen") + target = evidence_entry(manifest, evidence_id) + if target.get("authority") != producer or \ + target.get("format") != "normalized-jsonl": + raise HarnessError("target evidence declaration differs") + if target.get("upstream_evidence") != [raw_id]: + raise HarnessError("target evidence upstream declaration differs") + if draft: + if target.get("status") not in ("planned", "verified"): + raise HarnessError("target evidence draft status differs") + elif target.get("status") != "verified": + raise HarnessError("target evidence is not frozen") + return target + + +def build_context(arguments, producer, evidence_id, + allow_unpinned_descriptor=False, draft_evidence=False, + source_overrides=None, executed_harness=None): + repository_input_path = pathlib.Path(arguments.repository) + if repository_input_path.is_symlink(): + raise HarnessError("repository root is a symbolic link") + repository = repository_input_path.resolve(strict=True) + manifest_path = pathlib.Path(arguments.manifest).resolve(strict=True) + expected_manifest = repository_input(repository, + "test/conformance/n6/manifest.toml") + if manifest_path != expected_manifest: + raise HarnessError("manifest path does not select the repository input") + manifest, _, _ = load_toml_snapshot(expected_manifest, "N6 manifest") + expected_capabilities = repository_input(repository, + manifest["capabilities_file"]) + expected_fixtures = repository_input(repository, + manifest["fixture_manifest_file"]) + if pathlib.Path(arguments.capabilities).resolve(strict=True) != \ + expected_capabilities: + raise HarnessError("capability path does not select the frozen input") + if pathlib.Path(arguments.fixtures).resolve(strict=True) != expected_fixtures: + raise HarnessError("fixture path does not select the frozen input") + capabilities, _, capabilities_sha256 = load_toml_snapshot( + expected_capabilities, "capability manifest") + fixtures, _, fixtures_sha256 = load_toml_snapshot(expected_fixtures, + "fixture manifest") + authority_record = authority(capabilities, producer) + descriptor_path = pathlib.Path(arguments.descriptor).resolve(strict=True) + descriptor, _, descriptor_sha256 = load_toml_snapshot(descriptor_path, + "Python toolchain descriptor") + if executed_harness is not None: + if source_overrides is not None: + raise HarnessError("authenticated source overrides are ambiguous") + source_overrides = authenticated_source_overrides(repository, descriptor, + descriptor_path, executed_harness) + authorized = authority_record["toolchain_sha256"] + if allow_unpinned_descriptor: + if authorized or descriptor.get("status") != "planned": + raise HarnessError("draft descriptor mode is not authorized") + elif descriptor_sha256 not in authorized: + raise HarnessError("descriptor digest is not authorized") + descriptor_inputs = descriptor_repository_inputs(repository, descriptor, + source_overrides) + known = fixture_map(fixtures) + semantic, semantic_path = semantic_case_map(repository, manifest, + capabilities, fixtures) + overlap = set(known) & set(semantic) + if overlap: + raise HarnessError(f"semantic cases overlap fixtures: {sorted(overlap)}") + known.update(semantic) + claims = selected_claims(authority_record) + unknown = sorted({case_id for case_id, _ in claims} - set(known)) + if unknown: + raise HarnessError(f"authority claims unknown cases: {unknown}") + out_of_scope = sorted((case_id, capability) + for case_id, capability in claims + if capability not in known[case_id]["capabilities"]) + if out_of_scope: + raise HarnessError(f"authority claims out-of-scope cases: {out_of_scope}") + limits = manifest["evidence_limits"] + raw_entry = evidence_entry(manifest, "normalized-raw-java-apache-corpus") + raw_evidence_path = repository_input(repository, raw_entry["file"]) + if pathlib.Path(arguments.raw_evidence).resolve(strict=True) != \ + raw_evidence_path: + raise HarnessError("raw evidence path does not select the frozen input") + raw_run, raw_files, raw_columns, raw_evidence_sha256 = load_raw_facts( + raw_evidence_path, + limits["max_file_bytes"], limits["max_line_bytes"], + limits["max_records_per_input"]) + target_entry = _verify_evidence_bindings(manifest, producer, evidence_id, + raw_entry, raw_evidence_sha256, + allow_unpinned_descriptor or draft_evidence) + target_output = manifest_output_path(repository, target_entry, + arguments.output) + hashes = frozen_input_hashes(repository, manifest, { + "capabilities_sha256": capabilities_sha256, + "fixture_manifest_sha256": fixtures_sha256, + }) + for field, digest in hashes.items(): + if manifest.get(field) != digest: + raise HarnessError(f"manifest has stale {field}") + if raw_run.get(field) != digest: + raise HarnessError(f"normalized raw evidence has stale {field}") + raw_authority = authority(capabilities, "n6-raw-java") + if raw_run.get("evidence_id") != "normalized-raw-java-apache-corpus" or \ + raw_run.get("producer_version") != raw_authority["version"] or \ + raw_run.get("source_revision") != raw_authority["revision"] or \ + raw_run.get("toolchain_sha256") not in \ + raw_authority["toolchain_sha256"]: + raise HarnessError("normalized raw evidence has the wrong authority") + upstream = { + "evidence_id": raw_run["evidence_id"], + "file": raw_entry["file"], + "sha256": raw_evidence_sha256, + } + frozen_paths = frozen_input_paths(repository, manifest) + protected_inputs = [expected_manifest, descriptor_path, raw_evidence_path, + semantic_path, *frozen_paths.values(), *descriptor_inputs] + return { + "repository": repository, + "manifest": manifest, + "fixtures": fixtures, + "authority_record": authority_record, + "known": known, + "claims": claims, + "raw_files": raw_files, + "raw_columns": raw_columns, + "limits": limits, + "descriptor": descriptor, + "descriptor_sha256": descriptor_sha256, + "raw_evidence_path": raw_evidence_path, + "target_entry": target_entry, + "output_path": target_output, + "input_hashes": hashes, + "source_overrides": {} if source_overrides is None else source_overrides, + "descriptor_inputs": descriptor_inputs, + "upstream_evidence": upstream, + "protected_inputs": protected_inputs, + } diff --git a/test/conformance/n6/harnesses/duckdb.py b/test/conformance/n6/harnesses/duckdb.py new file mode 100644 index 0000000..ba48e40 --- /dev/null +++ b/test/conformance/n6/harnesses/duckdb.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import os +import pathlib +import stat +import subprocess +import sys +import tempfile +import tomllib + + +_SNAPSHOT_ENV = "PARQUET_N6_AUTHENTICATED_SNAPSHOT" +_SNAPSHOT_DESCRIPTOR = "descriptor.toml" +_SOURCE_LIMIT = 4 * 1024 * 1024 + + +def _source_identity(metadata): + return (metadata.st_dev, metadata.st_ino, metadata.st_mode, + metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, + metadata.st_ctime_ns) + + +def _source_bytes(path, label): + path = pathlib.Path(path) + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode) or \ + not 0 <= metadata.st_size <= _SOURCE_LIMIT: + raise RuntimeError(f"{label} is not a bounded regular file") + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb") as stream: + opened = os.fstat(stream.fileno()) + if _source_identity(opened) != _source_identity(metadata): + raise RuntimeError(f"{label} changed while it was opened") + value = stream.read(_SOURCE_LIMIT + 1) + final = os.fstat(stream.fileno()) + if len(value) != opened.st_size or len(value) > _SOURCE_LIMIT or \ + _source_identity(final) != _source_identity(opened) or \ + _source_identity(path.lstat()) != _source_identity(opened): + raise RuntimeError(f"{label} changed while it was read") + return value + + +def _named_argument(arguments, name): + matches = [index for index, value in enumerate(arguments) if value == name] + if len(matches) != 1 or matches[0] + 1 >= len(arguments): + raise RuntimeError(f"bootstrap argument is absent or repeated: {name}") + return arguments[matches[0] + 1], matches[0] + 1 + + +def _repository_source(repository, relative, label): + if not isinstance(relative, str) or "\\" in relative: + raise RuntimeError(f"{label} path is invalid") + pure = pathlib.PurePosixPath(relative) + if pure.is_absolute() or any(part in ("", ".", "..") for part in pure.parts): + raise RuntimeError(f"{label} path is invalid") + current = repository + for part in pure.parts: + current = current / part + metadata = current.lstat() + if stat.S_ISLNK(metadata.st_mode): + raise RuntimeError(f"{label} path contains a symbolic link") + if not current.is_file(): + raise RuntimeError(f"{label} is not a regular file") + return current + + +def _write_snapshot(path, value): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o400) + os.fchmod(descriptor, 0o400) + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + return + + +def _snapshot_sources(arguments, harness_file): + repository_value, _ = _named_argument(arguments, "--repository") + descriptor_value, descriptor_index = _named_argument(arguments, + "--descriptor") + repository_input = pathlib.Path(repository_value) + repository_metadata = repository_input.lstat() + if stat.S_ISLNK(repository_metadata.st_mode) or \ + not stat.S_ISDIR(repository_metadata.st_mode): + raise RuntimeError("repository root is not a regular directory") + repository = repository_input.resolve(strict=True) + descriptor_source = _source_bytes(descriptor_value, + "Python toolchain descriptor") + try: + descriptor = tomllib.loads(descriptor_source.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise RuntimeError("Python toolchain descriptor is invalid") from error + harness = _repository_source(repository, descriptor.get("harness_file"), + "harness source") + if harness.resolve(strict=True) != pathlib.Path(harness_file).resolve( + strict=True): + raise RuntimeError("executed harness does not match its descriptor") + common_relative = (pathlib.PurePosixPath( + descriptor["harness_file"]).parent / "common.py").as_posix() + support = [item for item in descriptor.get("support_files", []) + if isinstance(item, dict) and item.get("file") == common_relative] + if len(support) != 1 or not isinstance(support[0].get("sha256"), str): + raise RuntimeError("common source binding is absent or ambiguous") + common = _repository_source(repository, common_relative, "common source") + harness_source = _source_bytes(harness, "harness source") + common_source = _source_bytes(common, "common source") + if hashlib.sha256(harness_source).hexdigest() != \ + descriptor.get("harness_sha256"): + raise RuntimeError("harness source digest differs") + if hashlib.sha256(common_source).hexdigest() != support[0]["sha256"]: + raise RuntimeError("common source digest differs") + return (descriptor_source, descriptor_index, harness_source, + common_source) + + +def _verify_snapshot_child(arguments, harness_file): + root_input = pathlib.Path(os.environ[_SNAPSHOT_ENV]) + root_metadata = root_input.lstat() + if stat.S_ISLNK(root_metadata.st_mode) or \ + not stat.S_ISDIR(root_metadata.st_mode) or \ + stat.S_IMODE(root_metadata.st_mode) != 0o500: + raise RuntimeError("authenticated source snapshot is not read-only") + root = root_input.resolve(strict=True) + descriptor_value, _ = _named_argument(arguments, "--descriptor") + descriptor_path = root / _SNAPSHOT_DESCRIPTOR + harness_path = root / pathlib.Path(harness_file).name + common_path = root / "common.py" + if pathlib.Path(descriptor_value).resolve(strict=True) != descriptor_path or \ + pathlib.Path(harness_file).resolve(strict=True) != harness_path: + raise RuntimeError("authenticated source snapshot paths differ") + if {path.name for path in root.iterdir()} != { + descriptor_path.name, harness_path.name, common_path.name}: + raise RuntimeError("authenticated source snapshot contents differ") + for path in (descriptor_path, harness_path, common_path): + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode) or \ + not stat.S_ISREG(metadata.st_mode) or \ + stat.S_IMODE(metadata.st_mode) != 0o400: + raise RuntimeError("authenticated source snapshot file is writable") + descriptor_source = _source_bytes(descriptor_path, + "snapshotted Python toolchain descriptor") + try: + descriptor = tomllib.loads(descriptor_source.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise RuntimeError("snapshotted descriptor is invalid") from error + support = [item for item in descriptor.get("support_files", []) + if isinstance(item, dict) and pathlib.PurePosixPath( + item.get("file", "")).name == "common.py"] + if len(support) != 1 or hashlib.sha256(_source_bytes(harness_path, + "snapshotted harness source")).hexdigest() != \ + descriptor.get("harness_sha256") or hashlib.sha256(_source_bytes( + common_path, "snapshotted common source")).hexdigest() != \ + support[0].get("sha256"): + raise RuntimeError("authenticated source snapshot digest differs") + return + + +# The outer process uses only the standard library. It authenticates and freezes +# harness/common bytes before the isolated child imports common or reads evidence. +def _authenticated_launch(arguments, harness_file): + if _SNAPSHOT_ENV in os.environ: + _verify_snapshot_child(arguments, harness_file) + return None + sources = _snapshot_sources(arguments, harness_file) + descriptor_source, descriptor_index, harness_source, common_source = sources + with tempfile.TemporaryDirectory(prefix="parquet-n6-sources-") as directory: + root = pathlib.Path(directory).resolve(strict=True) + descriptor_path = root / _SNAPSHOT_DESCRIPTOR + harness_path = root / pathlib.Path(harness_file).name + _write_snapshot(descriptor_path, descriptor_source) + _write_snapshot(harness_path, harness_source) + _write_snapshot(root / "common.py", common_source) + os.chmod(root, 0o500) + child_arguments = list(arguments[1:]) + child_arguments[descriptor_index - 1] = str(descriptor_path) + environment = os.environ.copy() + environment[_SNAPSHOT_ENV] = str(root) + try: + result = subprocess.run([sys.executable, "-I", "-B", "-S", + str(harness_path), *child_arguments], check=False, + env=environment) + finally: + os.chmod(root, 0o700) + return result.returncode + + +if __name__ == "__main__": + try: + _launch_status = _authenticated_launch(sys.argv, __file__) + except Exception as error: + print(f"N6 DuckDB bootstrap failed: {error}", file=sys.stderr) + sys.exit(1) + if _launch_status is not None: + sys.exit(_launch_status) + +SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIRECTORY)) +from common import (HarnessError, atomic_output, build_context, case_result, + checked_file, evidence_bytes, exact_wheel_module, leaf_records, + logical_observations, + reject_output_alias, run_record, verify_python_descriptor) +sys.path = [entry for entry in sys.path + if pathlib.Path(entry or ".").resolve() != SCRIPT_DIRECTORY] + + +PRODUCER = "duckdb" +EVIDENCE_ID = "normalized-duckdb" + + +def decoded_columns(connection, path, case_id, raw_columns): + result = connection.execute("SELECT * FROM read_parquet(?)", [str(path)]) + names = [description[0] for description in result.description] + rows = result.fetchall() + leaves = leaf_records(case_id, raw_columns) + expected = [record["path"][0] for record in leaves] + if names != expected: + raise HarnessError(f"DuckDB column order differs: {case_id}") + columns = [[row[index] for row in rows] for index in range(len(names))] + return columns, len(rows) + + +def metadata_observations(connection, path): + selected = [ + "row_group_id", "row_group_num_rows", "row_group_num_columns", + "row_group_bytes", "column_id", "file_offset", "num_values", + "path_in_schema", "type", "stats_min", "stats_max", + "stats_null_count", "stats_distinct_count", "stats_min_value", + "stats_max_value", "compression", "encodings", "index_page_offset", + "dictionary_page_offset", "data_page_offset", "total_compressed_size", + "total_uncompressed_size", "bloom_filter_offset", + "bloom_filter_length", "min_is_exact", "max_is_exact", + "row_group_compressed_bytes", + ] + query = "SELECT " + ",".join(selected) + ( + " FROM parquet_metadata(?) ORDER BY row_group_id, column_id") + rows = connection.execute(query, [str(path)]).fetchall() + observations = [dict(zip(selected, row)) for row in rows] + return [{ + "contract": "n6-duckdb-parquet-metadata-v1", + "rows": observations, + }] + + +def generate(arguments, context, duckdb_runtime): + repository = context["repository"] + manifest = context["manifest"] + authority_record = context["authority_record"] + known = context["known"] + claims = context["claims"] + raw_files = context["raw_files"] + raw_columns = context["raw_columns"] + limits = context["limits"] + descriptor_sha256 = context["descriptor_sha256"] + unsupported = {case_id for (case_id, _), status in claims.items() + if status == "unsupported"} + records = [run_record(EVIDENCE_ID, PRODUCER, authority_record, + descriptor_sha256, repository, manifest, unsupported, + context["upstream_evidence"], input_hashes=context["input_hashes"])] + by_case = {} + for (case_id, capability), status in claims.items(): + by_case.setdefault(case_id, []).append((capability, status)) + connection = duckdb_runtime.connect(":memory:") + try: + with tempfile.TemporaryDirectory(prefix="parquet-n6-duckdb-") as snapshots: + for case_id in sorted(by_case): + fixture = known[case_id] + if case_id not in raw_files: + raise HarnessError(f"raw file fact is absent: {case_id}") + path = checked_file(arguments.corpus_root, fixture["file"], + fixture["sha256"], fixture["size"], snapshots) + records.append(raw_files[case_id]) + logical = None + metadata = None + for capability, status in sorted(by_case[case_id]): + if status == "unsupported": + result_status = "UNSUPPORTED" + observations = None + elif capability == "read.logical-values": + if logical is None: + columns, rows = decoded_columns(connection, path, + case_id, raw_columns) + logical = logical_observations(case_id, raw_columns, + columns, rows) + result_status = "PASS" + observations = logical + elif capability == "read.parquet-metadata-view": + if metadata is None: + metadata = metadata_observations(connection, path) + result_status = "PASS" + observations = metadata + else: + raise HarnessError( + f"unhandled DuckDB capability: {capability}") + records.append(case_result(case_id, capability, + fixture["digest_contract"], result_status, + observations)) + finally: + connection.close() + return evidence_bytes(records, limits["max_file_bytes"], + limits["max_line_bytes"], limits["max_records_per_input"]) + + +def parser(): + result = argparse.ArgumentParser() + result.add_argument("--repository", required=True) + result.add_argument("--manifest", required=True) + result.add_argument("--capabilities", required=True) + result.add_argument("--fixtures", required=True) + result.add_argument("--descriptor", required=True) + result.add_argument("--raw-evidence", required=True) + result.add_argument("--wheel", required=True) + result.add_argument("--corpus-root", required=True) + result.add_argument("--output", required=True) + result.add_argument("--check", action="store_true") + result.add_argument("--draft", action="store_true") + return result + + +def main(): + arguments = parser().parse_args() + context = build_context(arguments, PRODUCER, EVIDENCE_ID, + draft_evidence=arguments.draft, executed_harness=__file__) + verify_python_descriptor(context["descriptor"], context["repository"], + PRODUCER, context["source_overrides"], context["descriptor_inputs"]) + reject_output_alias(context["output_path"], + [*context["protected_inputs"], arguments.wheel], + [arguments.corpus_root]) + with exact_wheel_module(arguments.wheel, context["descriptor"], + "duckdb", context["authority_record"]["version"]) as runtime: + value = generate(arguments, context, runtime) + atomic_output(context["output_path"], value, arguments.check) + print(f"{EVIDENCE_ID}: {len(value)} bytes") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as error: + print(f"N6 DuckDB harness failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/test/conformance/n6/harnesses/pyarrow.py b/test/conformance/n6/harnesses/pyarrow.py new file mode 100644 index 0000000..8fb22db --- /dev/null +++ b/test/conformance/n6/harnesses/pyarrow.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import importlib +import os +import pathlib +import stat +import subprocess +import sys +import tempfile +import tomllib + + +_SNAPSHOT_ENV = "PARQUET_N6_AUTHENTICATED_SNAPSHOT" +_SNAPSHOT_DESCRIPTOR = "descriptor.toml" +_SOURCE_LIMIT = 4 * 1024 * 1024 + + +def _source_identity(metadata): + return (metadata.st_dev, metadata.st_ino, metadata.st_mode, + metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, + metadata.st_ctime_ns) + + +def _source_bytes(path, label): + path = pathlib.Path(path) + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode) or \ + not 0 <= metadata.st_size <= _SOURCE_LIMIT: + raise RuntimeError(f"{label} is not a bounded regular file") + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb") as stream: + opened = os.fstat(stream.fileno()) + if _source_identity(opened) != _source_identity(metadata): + raise RuntimeError(f"{label} changed while it was opened") + value = stream.read(_SOURCE_LIMIT + 1) + final = os.fstat(stream.fileno()) + if len(value) != opened.st_size or len(value) > _SOURCE_LIMIT or \ + _source_identity(final) != _source_identity(opened) or \ + _source_identity(path.lstat()) != _source_identity(opened): + raise RuntimeError(f"{label} changed while it was read") + return value + + +def _named_argument(arguments, name): + matches = [index for index, value in enumerate(arguments) if value == name] + if len(matches) != 1 or matches[0] + 1 >= len(arguments): + raise RuntimeError(f"bootstrap argument is absent or repeated: {name}") + return arguments[matches[0] + 1], matches[0] + 1 + + +def _repository_source(repository, relative, label): + if not isinstance(relative, str) or "\\" in relative: + raise RuntimeError(f"{label} path is invalid") + pure = pathlib.PurePosixPath(relative) + if pure.is_absolute() or any(part in ("", ".", "..") for part in pure.parts): + raise RuntimeError(f"{label} path is invalid") + current = repository + for part in pure.parts: + current = current / part + metadata = current.lstat() + if stat.S_ISLNK(metadata.st_mode): + raise RuntimeError(f"{label} path contains a symbolic link") + if not current.is_file(): + raise RuntimeError(f"{label} is not a regular file") + return current + + +def _write_snapshot(path, value): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o400) + os.fchmod(descriptor, 0o400) + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + return + + +def _snapshot_sources(arguments, harness_file): + repository_value, _ = _named_argument(arguments, "--repository") + descriptor_value, descriptor_index = _named_argument(arguments, + "--descriptor") + repository_input = pathlib.Path(repository_value) + repository_metadata = repository_input.lstat() + if stat.S_ISLNK(repository_metadata.st_mode) or \ + not stat.S_ISDIR(repository_metadata.st_mode): + raise RuntimeError("repository root is not a regular directory") + repository = repository_input.resolve(strict=True) + descriptor_source = _source_bytes(descriptor_value, + "Python toolchain descriptor") + try: + descriptor = tomllib.loads(descriptor_source.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise RuntimeError("Python toolchain descriptor is invalid") from error + harness = _repository_source(repository, descriptor.get("harness_file"), + "harness source") + if harness.resolve(strict=True) != pathlib.Path(harness_file).resolve( + strict=True): + raise RuntimeError("executed harness does not match its descriptor") + common_relative = (pathlib.PurePosixPath( + descriptor["harness_file"]).parent / "common.py").as_posix() + support = [item for item in descriptor.get("support_files", []) + if isinstance(item, dict) and item.get("file") == common_relative] + if len(support) != 1 or not isinstance(support[0].get("sha256"), str): + raise RuntimeError("common source binding is absent or ambiguous") + common = _repository_source(repository, common_relative, "common source") + harness_source = _source_bytes(harness, "harness source") + common_source = _source_bytes(common, "common source") + if hashlib.sha256(harness_source).hexdigest() != \ + descriptor.get("harness_sha256"): + raise RuntimeError("harness source digest differs") + if hashlib.sha256(common_source).hexdigest() != support[0]["sha256"]: + raise RuntimeError("common source digest differs") + return (descriptor_source, descriptor_index, harness_source, + common_source) + + +def _verify_snapshot_child(arguments, harness_file): + root_input = pathlib.Path(os.environ[_SNAPSHOT_ENV]) + root_metadata = root_input.lstat() + if stat.S_ISLNK(root_metadata.st_mode) or \ + not stat.S_ISDIR(root_metadata.st_mode) or \ + stat.S_IMODE(root_metadata.st_mode) != 0o500: + raise RuntimeError("authenticated source snapshot is not read-only") + root = root_input.resolve(strict=True) + descriptor_value, _ = _named_argument(arguments, "--descriptor") + descriptor_path = root / _SNAPSHOT_DESCRIPTOR + harness_path = root / pathlib.Path(harness_file).name + common_path = root / "common.py" + if pathlib.Path(descriptor_value).resolve(strict=True) != descriptor_path or \ + pathlib.Path(harness_file).resolve(strict=True) != harness_path: + raise RuntimeError("authenticated source snapshot paths differ") + if {path.name for path in root.iterdir()} != { + descriptor_path.name, harness_path.name, common_path.name}: + raise RuntimeError("authenticated source snapshot contents differ") + for path in (descriptor_path, harness_path, common_path): + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode) or \ + not stat.S_ISREG(metadata.st_mode) or \ + stat.S_IMODE(metadata.st_mode) != 0o400: + raise RuntimeError("authenticated source snapshot file is writable") + descriptor_source = _source_bytes(descriptor_path, + "snapshotted Python toolchain descriptor") + try: + descriptor = tomllib.loads(descriptor_source.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise RuntimeError("snapshotted descriptor is invalid") from error + support = [item for item in descriptor.get("support_files", []) + if isinstance(item, dict) and pathlib.PurePosixPath( + item.get("file", "")).name == "common.py"] + if len(support) != 1 or hashlib.sha256(_source_bytes(harness_path, + "snapshotted harness source")).hexdigest() != \ + descriptor.get("harness_sha256") or hashlib.sha256(_source_bytes( + common_path, "snapshotted common source")).hexdigest() != \ + support[0].get("sha256"): + raise RuntimeError("authenticated source snapshot digest differs") + return + + +# The outer process uses only the standard library. It authenticates and freezes +# harness/common bytes before the isolated child imports common or reads evidence. +def _authenticated_launch(arguments, harness_file): + if _SNAPSHOT_ENV in os.environ: + _verify_snapshot_child(arguments, harness_file) + return None + sources = _snapshot_sources(arguments, harness_file) + descriptor_source, descriptor_index, harness_source, common_source = sources + with tempfile.TemporaryDirectory(prefix="parquet-n6-sources-") as directory: + root = pathlib.Path(directory).resolve(strict=True) + descriptor_path = root / _SNAPSHOT_DESCRIPTOR + harness_path = root / pathlib.Path(harness_file).name + _write_snapshot(descriptor_path, descriptor_source) + _write_snapshot(harness_path, harness_source) + _write_snapshot(root / "common.py", common_source) + os.chmod(root, 0o500) + child_arguments = list(arguments[1:]) + child_arguments[descriptor_index - 1] = str(descriptor_path) + environment = os.environ.copy() + environment[_SNAPSHOT_ENV] = str(root) + try: + result = subprocess.run([sys.executable, "-I", "-B", "-S", + str(harness_path), *child_arguments], check=False, + env=environment) + finally: + os.chmod(root, 0o700) + return result.returncode + + +if __name__ == "__main__": + try: + _launch_status = _authenticated_launch(sys.argv, __file__) + except Exception as error: + print(f"N6 PyArrow bootstrap failed: {error}", file=sys.stderr) + sys.exit(1) + if _launch_status is not None: + sys.exit(_launch_status) + +SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIRECTORY)) +from common import (HarnessError, atomic_output, build_context, case_result, + canonical_value, checked_file, evidence_bytes, leaf_records, + exact_wheel_module, logical_observations, reject_output_alias, run_record, + verify_python_descriptor) +sys.path = [entry for entry in sys.path + if pathlib.Path(entry or ".").resolve() != SCRIPT_DIRECTORY] + + +PRODUCER = "pyarrow" +EVIDENCE_ID = "normalized-pyarrow" + + +def decoded_columns(parquet_runtime, path, case_id, raw_columns): + table = parquet_runtime.read_table(path) + leaves = leaf_records(case_id, raw_columns) + names = list(table.column_names) + expected = [record["path"][0] for record in leaves] + if names != expected: + raise HarnessError(f"PyArrow column order differs: {case_id}") + return [table.column(name).to_pylist() for name in names], table.num_rows + + +def statistics_observations(parquet_runtime, path, case_id, raw_columns): + parquet = parquet_runtime.ParquetFile(path) + leaves = leaf_records(case_id, raw_columns) + row_groups = [] + for row_group in range(parquet.metadata.num_row_groups): + columns = [] + metadata = parquet.metadata.row_group(row_group) + if metadata.num_columns != len(leaves): + raise HarnessError(f"PyArrow statistics leaf count differs: {case_id}") + for leaf, column in zip(leaves, (metadata.column(index) + for index in range(metadata.num_columns))): + statistics = column.statistics + columns.append({ + "distinct_count": None if statistics is None else + statistics.distinct_count, + "has_min_max": False if statistics is None else + statistics.has_min_max, + "leaf": leaf["leaf"], + "max": None if statistics is None or + not statistics.has_min_max else + canonical_value(statistics.max, leaf["leaf_schema"]), + "min": None if statistics is None or + not statistics.has_min_max else + canonical_value(statistics.min, leaf["leaf_schema"]), + "null_count": None if statistics is None else + statistics.null_count, + "path": leaf["path"], + }) + row_groups.append({"columns": columns, "row_group": row_group}) + return [{ + "contract": "n6-pyarrow-statistics-v1", + "row_groups": row_groups, + }] + + +def generate(arguments, context, parquet_runtime): + repository = context["repository"] + manifest = context["manifest"] + authority_record = context["authority_record"] + known = context["known"] + claims = context["claims"] + raw_files = context["raw_files"] + raw_columns = context["raw_columns"] + limits = context["limits"] + descriptor_sha256 = context["descriptor_sha256"] + unsupported = {case_id for (case_id, _), status in claims.items() + if status == "unsupported"} + records = [run_record(EVIDENCE_ID, PRODUCER, authority_record, + descriptor_sha256, repository, manifest, unsupported, + context["upstream_evidence"], input_hashes=context["input_hashes"])] + by_case = {} + for (case_id, capability), status in claims.items(): + by_case.setdefault(case_id, []).append((capability, status)) + with tempfile.TemporaryDirectory(prefix="parquet-n6-pyarrow-") as snapshots: + for case_id in sorted(by_case): + fixture = known[case_id] + if case_id not in raw_files: + raise HarnessError(f"raw file fact is absent: {case_id}") + path = checked_file(arguments.corpus_root, fixture["file"], + fixture["sha256"], fixture["size"], snapshots) + records.append(raw_files[case_id]) + logical = None + statistics = None + for capability, status in sorted(by_case[case_id]): + if status == "unsupported": + result_status = "UNSUPPORTED" + observations = None + elif capability == "read.logical-values": + if logical is None: + columns, rows = decoded_columns(parquet_runtime, path, + case_id, + raw_columns) + logical = logical_observations(case_id, raw_columns, + columns, rows) + result_status = "PASS" + observations = logical + elif capability == "read.statistics-metadata": + if statistics is None: + statistics = statistics_observations(parquet_runtime, + path, case_id, raw_columns) + result_status = "PASS" + observations = statistics + else: + raise HarnessError( + f"unhandled PyArrow capability: {capability}") + records.append(case_result(case_id, capability, + fixture["digest_contract"], result_status, observations)) + return evidence_bytes(records, limits["max_file_bytes"], + limits["max_line_bytes"], limits["max_records_per_input"]) + + +def parser(): + result = argparse.ArgumentParser() + result.add_argument("--repository", required=True) + result.add_argument("--manifest", required=True) + result.add_argument("--capabilities", required=True) + result.add_argument("--fixtures", required=True) + result.add_argument("--descriptor", required=True) + result.add_argument("--raw-evidence", required=True) + result.add_argument("--wheel", required=True) + result.add_argument("--corpus-root", required=True) + result.add_argument("--output", required=True) + result.add_argument("--check", action="store_true") + result.add_argument("--draft", action="store_true") + return result + + +def main(): + arguments = parser().parse_args() + context = build_context(arguments, PRODUCER, EVIDENCE_ID, + draft_evidence=arguments.draft, executed_harness=__file__) + verify_python_descriptor(context["descriptor"], context["repository"], + PRODUCER, context["source_overrides"], context["descriptor_inputs"]) + reject_output_alias(context["output_path"], + [*context["protected_inputs"], arguments.wheel], + [arguments.corpus_root]) + with exact_wheel_module(arguments.wheel, context["descriptor"], + "pyarrow", context["authority_record"]["version"]): + parquet_runtime = importlib.import_module("pyarrow.parquet") + value = generate(arguments, context, parquet_runtime) + atomic_output(context["output_path"], value, arguments.check) + print(f"{EVIDENCE_ID}: {len(value)} bytes") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as error: + print(f"N6 PyArrow harness failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/test/conformance/n6/harnesses/test_common.py b/test/conformance/n6/harnesses/test_common.py new file mode 100644 index 0000000..d8b1349 --- /dev/null +++ b/test/conformance/n6/harnesses/test_common.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +import datetime +import decimal +import hashlib +import os +import pathlib +import platform +import stat +import subprocess +import sys +import tempfile +import textwrap +import types +import unittest +import zipfile + +SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIRECTORY)) +import common + + +class CommonHarnessTests(unittest.TestCase): + def assert_harness_error(self, function, *arguments): + with self.assertRaises(common.HarnessError): + function(*arguments) + + def test_safe_relative_paths(self): + self.assertTrue(common.safe_relative("data/a-b_1.parquet")) + for value in ("", "/tmp/file", "../file", "data//file", + "data/./file", "data/../file", "data\\file", "data/é"): + self.assertFalse(common.safe_relative(value), value) + + def test_toml_snapshot_binds_values_and_digest_to_one_read(self): + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "control.toml" + original = b'version = "one"\n' + path.write_bytes(original) + value, source, digest = common.load_toml_snapshot(path) + path.write_bytes(b'version = "two"\n') + self.assertEqual(value, {"version": "one"}) + self.assertEqual(source, original) + self.assertEqual(digest, hashlib.sha256(original).hexdigest()) + + def test_decimal_values_are_exact(self): + self.assertEqual(common.decimal_unscaled(decimal.Decimal("1.230"), 2), + "123") + self.assertEqual(common.decimal_unscaled(decimal.Decimal("-1.23"), 2), + "-123") + value = decimal.Decimal("12345678901234567890123456789012345678") + self.assertEqual(common.decimal_unscaled(value, 0), str(value)) + self.assert_harness_error(common.decimal_unscaled, + decimal.Decimal("0.001"), 2) + self.assert_harness_error(common.decimal_unscaled, + decimal.Decimal("Infinity"), 2) + + def test_timestamps_use_utc_instants(self): + offset = datetime.timezone(datetime.timedelta(hours=1)) + value = datetime.datetime(1970, 1, 1, 1, tzinfo=offset) + self.assertEqual(common.epoch_nanoseconds(value), 0) + value = datetime.datetime(1970, 1, 1, 0, 0, 0, 1) + self.assertEqual(common.epoch_nanoseconds(value), 1000) + + def test_checked_fixture_rejects_links_and_identity_changes(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + fixture = root / "fixture.parquet" + fixture.write_bytes(b"PAR1testPAR1") + digest = hashlib.sha256(fixture.read_bytes()).hexdigest() + snapshots = root / "snapshots" + snapshots.mkdir() + snapshot = common.checked_file(root, "fixture.parquet", digest, + fixture.stat().st_size, snapshots) + self.assertEqual(snapshot.read_bytes(), fixture.read_bytes()) + self.assertEqual(stat.S_IMODE(snapshot.stat().st_mode), 0o600) + fixture.write_bytes(b"PAR1new!PAR1") + self.assertNotEqual(snapshot.read_bytes(), fixture.read_bytes()) + fixture.write_bytes(b"PAR1testPAR1") + self.assert_harness_error(common.checked_file, root, + "fixture.parquet", "0" * 64, fixture.stat().st_size, + snapshots) + link = root / "linked.parquet" + link.symlink_to(fixture) + self.assert_harness_error(common.checked_file, root, + "linked.parquet", digest, fixture.stat().st_size, snapshots) + root_link = root.parent / f"{root.name}-link" + root_link.symlink_to(root, target_is_directory=True) + try: + self.assert_harness_error(common.checked_file, root_link, + "fixture.parquet", digest, fixture.stat().st_size, + snapshots) + finally: + root_link.unlink() + + def test_raw_evidence_requires_canonical_bounded_jsonl(self): + run = { + "producer": "n6-raw-java", + "record": "run", + "schema_version": 2, + } + file_record = {"case_id": "case", "record": "file"} + column = { + "case_id": "case", + "leaf": 0, + "record": "column_statistics", + "row_group": 0, + } + result = {"case_id": "case", "record": "case_result"} + records = [run, file_record, column, result] + value = common.evidence_bytes(records, 4096, 4096, 8) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "raw.jsonl" + path.write_bytes(value) + loaded_run, files, columns, digest = common.load_raw_facts(path, + 4096, 4096, 8) + self.assertEqual(loaded_run, run) + self.assertEqual(files["case"], file_record) + self.assertEqual(columns[("case", 0, 0)], column) + self.assertEqual(digest, hashlib.sha256(value).hexdigest()) + path.write_bytes(value.replace(b'"producer":', b'"producer" :', 1)) + self.assert_harness_error(common.load_raw_facts, path, 4096, 4096, + 8) + path.write_bytes(b"[]\n") + self.assert_harness_error(common.load_raw_facts, path, 4096, 4096, + 8) + path.write_bytes(b'{"record":"unknown"}\n') + self.assert_harness_error(common.load_raw_facts, path, 4096, 4096, + 8) + path.write_bytes(b'{"record":"run","record":"run"}\n') + self.assert_harness_error(common.load_raw_facts, path, 4096, 4096, + 8) + path.write_bytes(common.canonical_json(run).encode("utf-8")) + self.assert_harness_error(common.load_raw_facts, path, 4096, 4096, + 8) + path.write_bytes(value) + self.assert_harness_error(common.load_raw_facts, path, + len(value) - 1, 4096, 8) + self.assert_harness_error(common.load_raw_facts, path, 4096, + len(value.splitlines()[0]), 8) + self.assert_harness_error(common.load_raw_facts, path, 4096, 4096, + 3) + + def test_atomic_output_is_bounded_and_link_safe(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + output = root / "nested" / "evidence.jsonl" + value = b'{"record":"run"}\n' + common.atomic_output(output, value, False) + self.assertEqual(output.read_bytes(), value) + self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o644) + common.atomic_output(output, value, True) + output.write_bytes(value + b"stale") + self.assert_harness_error(common.atomic_output, output, value, True) + missing = root / "missing" / "evidence.jsonl" + self.assert_harness_error(common.atomic_output, missing, value, True) + self.assertFalse(missing.parent.exists()) + target = root / "target" + target.mkdir() + link = root / "parent-link" + link.symlink_to(target, target_is_directory=True) + linked_output = link / "evidence.jsonl" + common.atomic_output(linked_output, value, False) + self.assertEqual((target / "evidence.jsonl").read_bytes(), value) + destination_link = root / "destination-link" + destination_link.symlink_to(output) + self.assert_harness_error(common.atomic_output, destination_link, + value, False) + + def test_evidence_bindings_require_exact_upstream_and_digest(self): + digest = "1" * 64 + raw = { + "authority": "n6-raw-java", + "format": "normalized-jsonl", + "id": "normalized-raw-java-apache-corpus", + "sha256": digest, + "status": "verified", + } + target = { + "authority": "pyarrow", + "format": "normalized-jsonl", + "id": "normalized-pyarrow", + "status": "verified", + "upstream_evidence": ["normalized-raw-java-apache-corpus"], + } + manifest = {"frozen_evidence": [raw, target]} + self.assertEqual(common._verify_evidence_bindings(manifest, + "pyarrow", "normalized-pyarrow", raw, digest, False), target) + raw["sha256"] = "2" * 64 + self.assert_harness_error(common._verify_evidence_bindings, manifest, + "pyarrow", "normalized-pyarrow", raw, digest, False) + raw["sha256"] = digest + target["upstream_evidence"] = [] + self.assert_harness_error(common._verify_evidence_bindings, manifest, + "pyarrow", "normalized-pyarrow", raw, digest, False) + target["upstream_evidence"] = [ + "normalized-raw-java-apache-corpus"] + raw.pop("sha256") + raw["status"] = "planned" + target["status"] = "planned" + manifest = {"planned_evidence": [raw, target]} + common._verify_evidence_bindings(manifest, "pyarrow", + "normalized-pyarrow", raw, digest, True) + self.assert_harness_error(common._verify_evidence_bindings, manifest, + "pyarrow", "normalized-pyarrow", raw, digest, False) + + def test_repository_inputs_reject_links_and_escapes(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "input.toml" + source.write_text("version = 1\n") + self.assertEqual(common.repository_input(root, "input.toml"), + source.resolve()) + link = root / "link.toml" + link.symlink_to(source) + self.assert_harness_error(common.repository_input, root, + "link.toml") + self.assert_harness_error(common.repository_input, root, + "../input.toml") + + def test_descriptor_sources_are_exact(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + harness = root / "harness.py" + support = root / "common.py" + test = root / "test_common.py" + for path, value in ((harness, b"harness\n"), + (support, b"support\n"), (test, b"test\n")): + path.write_bytes(value) + descriptor = { + "harness_file": harness.name, + "harness_sha256": common.sha256_file(harness), + "support_files": [{ + "file": support.name, + "sha256": common.sha256_file(support), + }], + "test_file": test.name, + "test_sha256": common.sha256_file(test), + } + paths = common.descriptor_repository_inputs(root, descriptor) + self.assertEqual(set(paths), {path.resolve() + for path in (harness, support, test)}) + descriptor["support_files"][0]["sha256"] = "0" * 64 + self.assert_harness_error(common.descriptor_repository_inputs, + root, descriptor) + + def test_descriptor_source_override_authenticates_executed_snapshot(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + harness = root / "harness.py" + support = root / "common.py" + test = root / "test_common.py" + snapshot = root / "snapshot.py" + for path, value in ((harness, b"old harness\n"), + (snapshot, b"old harness\n"), (support, b"support\n"), + (test, b"test\n")): + path.write_bytes(value) + descriptor = { + "harness_file": harness.name, + "harness_sha256": common.sha256_file(harness), + "support_files": [{ + "file": support.name, + "sha256": common.sha256_file(support), + }], + "test_file": test.name, + "test_sha256": common.sha256_file(test), + } + harness.write_bytes(b"changed canonical harness\n") + paths = common.descriptor_repository_inputs(root, descriptor, + {harness.name: snapshot}) + self.assertIn(snapshot.resolve(), paths) + self.assertNotIn(harness.resolve(), paths) + + def test_python_descriptor_requires_distribution_contract(self): + descriptor = { + "authority": "tool", + "harness_file": "harness.py", + "harness_sha256": "0" * 64, + "harness_status": "planned", + "platform": "macos-15-arm64", + "python_distribution_sha256": "0" * 64, + "python_distribution_url": "https://example.invalid/python.tar.gz", + "python_executable_sha256": "0" * 64, + "python_tree_policy": "extract-strip-site-packages-bytecode-v1", + "python_tree_sha256": "0" * 64, + "python_version": "3.12.8", + "status": "planned", + "support_files": [], + "test_file": "test.py", + "test_sha256": "0" * 64, + "wheels": [], + } + for field in ("python_distribution_url", + "python_distribution_sha256", "python_tree_policy"): + incomplete = dict(descriptor) + incomplete.pop(field) + self.assert_harness_error(common.verify_python_descriptor, + incomplete, ".", "tool") + + def test_output_cannot_alias_inputs_or_corpus(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "source.jsonl" + source.write_text("{}\n") + corpus = root / "corpus" + corpus.mkdir() + self.assert_harness_error(common.reject_output_alias, source, + [source], [corpus]) + self.assert_harness_error(common.reject_output_alias, + corpus / "evidence.jsonl", [source], [corpus]) + common.reject_output_alias(root / "evidence.jsonl", [source], + [corpus]) + alias = root / "hardlink.jsonl" + alias.hardlink_to(source) + self.assert_harness_error(common.reject_output_alias, alias, + [source], [corpus]) + + def test_output_must_equal_manifest_target(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory).resolve() + parent = root / "test" / "conformance" / "n6" / "evidence" + parent.mkdir(parents=True) + target = { + "file": "test/conformance/n6/evidence/tool.normalized.jsonl", + } + expected = parent / "tool.normalized.jsonl" + self.assertEqual(common.manifest_output_path(root, target, + expected), expected) + self.assert_harness_error(common.manifest_output_path, root, + target, root / "other.jsonl") + alias = root / "evidence-link" + alias.symlink_to(parent, target_is_directory=True) + self.assert_harness_error(common.manifest_output_path, root, + target, alias / expected.name) + + def test_evidence_bytes_enforces_limits(self): + record = {"record": "run", "schema_version": 2} + value = common.evidence_bytes([record], 128, 128, 1) + self.assertTrue(value.endswith(b"\n")) + self.assert_harness_error(common.evidence_bytes, [], 128, 128, 1) + self.assert_harness_error(common.evidence_bytes, [record, record], + 256, 128, 1) + self.assert_harness_error(common.evidence_bytes, [record], 128, 8, 1) + self.assert_harness_error(common.evidence_bytes, [record], 8, 128, 1) + + def test_wheel_import_root_requires_exact_safe_bytes(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + wheel = root / "demo-1.0-cp312-cp312-macosx_12_0_arm64.whl" + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("demo/__init__.py", "version = '1.0'\n") + archive.writestr("demo-1.0.dist-info/RECORD", "") + archive.writestr("demo-1.0.dist-info/WHEEL", "Wheel-Version: 1.0\n") + descriptor = {"wheels": [{ + "name": wheel.name, + "sha256": common.sha256_file(wheel), + }]} + with common.wheel_import_root(wheel, descriptor) as extracted: + self.assertEqual((extracted / "demo" / "__init__.py").read_text(), + "version = '1.0'\n") + wheel.write_bytes(wheel.read_bytes() + b"changed") + with self.assertRaises(common.HarnessError): + with common.wheel_import_root(wheel, descriptor): + pass + + unsafe = root / "unsafe.whl" + with zipfile.ZipFile(unsafe, "w") as archive: + archive.writestr("../escape", "no") + archive.writestr("unsafe.dist-info/RECORD", "") + archive.writestr("unsafe.dist-info/WHEEL", "Wheel-Version: 1.0\n") + unsafe_descriptor = {"wheels": [{ + "name": unsafe.name, + "sha256": common.sha256_file(unsafe), + }]} + with self.assertRaises(common.HarnessError): + with common.wheel_import_root(unsafe, unsafe_descriptor): + pass + + def test_exact_wheel_module_owns_origins_and_cleans_all_modules(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + wheel = root / "duckdb-1.0-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("duckdb/__init__.py", + "import _duckdb\n__version__ = '1.0'\n") + archive.writestr("_duckdb.py", "value = 1\n") + archive.writestr("duckdb-1.0.dist-info/RECORD", "") + archive.writestr("duckdb-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\n") + descriptor = {"wheels": [{ + "name": wheel.name, + "sha256": common.sha256_file(wheel), + }]} + with common.exact_wheel_module(wheel, descriptor, "duckdb", + "1.0") as module: + self.assertEqual(module.__version__, "1.0") + self.assertIn("_duckdb", sys.modules) + self.assertNotIn("duckdb", sys.modules) + self.assertNotIn("_duckdb", sys.modules) + with self.assertRaises(common.HarnessError): + with common.exact_wheel_module(wheel, descriptor, "duckdb", + "0.0"): + pass + self.assertNotIn("duckdb", sys.modules) + self.assertNotIn("_duckdb", sys.modules) + for name in ("duckdb", "_duckdb"): + sys.modules[name] = types.ModuleType(name) + try: + with self.assertRaises(common.HarnessError): + with common.exact_wheel_module(wheel, descriptor, + "duckdb", "1.0"): + pass + self.assertIsInstance(sys.modules[name], types.ModuleType) + finally: + del sys.modules[name] + with self.assertRaises(common.HarnessError): + with common.exact_wheel_module(wheel, descriptor, "duckdb", + "1.0"): + sys.modules["duckdb.injected"] = types.ModuleType( + "duckdb.injected") + self.assertNotIn("duckdb", sys.modules) + self.assertNotIn("duckdb.injected", sys.modules) + self.assertNotIn("_duckdb", sys.modules) + + def test_wheel_owned_modules_include_native_extension_names(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "duckdb").mkdir() + (root / "duckdb" / "__init__.py").write_text("") + (root / "_duckdb.cpython-312-darwin.so").write_bytes(b"") + (root / "duckdb.libs").mkdir() + self.assertEqual(common._wheel_owned_modules(root), + {"duckdb", "_duckdb"}) + + def test_exact_wheel_module_rejects_an_outside_owned_origin(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + outside = root / "outside" + outside.mkdir() + (outside / "_demo.py").write_text("value = 1\n") + wheel = root / "demo-1.0-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("demo/__init__.py", + "import sys\nsys.path.pop(0)\nimport _demo\n" + "__version__ = '1.0'\n") + archive.writestr("_demo.py", "value = 2\n") + archive.writestr("demo-1.0.dist-info/RECORD", "") + archive.writestr("demo-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\n") + descriptor = {"wheels": [{ + "name": wheel.name, + "sha256": common.sha256_file(wheel), + }]} + sys.path.insert(0, str(outside)) + try: + with self.assertRaises(common.HarnessError): + with common.exact_wheel_module(wheel, descriptor, "demo", + "1.0"): + pass + finally: + sys.path.remove(str(outside)) + self.assertNotIn("demo", sys.modules) + self.assertNotIn("_demo", sys.modules) + + def test_python_runtime_matches_descriptor(self): + macos = platform.mac_ver()[0].split(".", 1)[0] + descriptor = { + "platform": f"macos-{macos}-{platform.machine()}", + "python_distribution_sha256": "1" * 64, + "python_distribution_url": + "https://github.com/astral-sh/python-build-standalone/" + "releases/download/example/python.tar.gz", + "python_executable_sha256": common.sha256_file( + pathlib.Path(sys.executable).resolve(strict=True)), + "python_tree_sha256": common.tree_sha256(sys.base_prefix), + "python_tree_policy": "extract-strip-site-packages-bytecode-v1", + "python_version": ".".join(str(value) + for value in sys.version_info[:3]), + } + if common.runtime_tree_policy_violations(sys.base_prefix): + self.assert_harness_error(common.verify_python_runtime, descriptor) + else: + common.verify_python_runtime(descriptor) + descriptor["python_version"] = "0.0.0" + self.assert_harness_error(common.verify_python_runtime, descriptor) + + def test_python_runtime_policy_rejects_generated_and_installed_files(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + library = root / "lib" / "python3.12" + library.mkdir(parents=True) + (library / "module.py").write_text("value = 1\n") + self.assertEqual(common.runtime_tree_policy_violations(root), []) + site = library / "site-packages" + site.mkdir() + self.assertTrue(common.runtime_tree_policy_violations(root)) + site.rmdir() + cache = library / "__pycache__" + cache.mkdir() + self.assertTrue(common.runtime_tree_policy_violations(root)) + cache.rmdir() + bytecode = library / "module.pyc" + bytecode.write_bytes(b"generated") + self.assertTrue(common.runtime_tree_policy_violations(root)) + + def test_python_runtime_rejects_missing_isolation_flags(self): + path = pathlib.Path(common.__file__).resolve() + version = ".".join(str(value) for value in sys.version_info[:3]) + template = textwrap.dedent(""" + import importlib.util + import pathlib + import sys + specification = importlib.util.spec_from_file_location( + "tested_common", pathlib.Path({path!r})) + module = importlib.util.module_from_spec(specification) + specification.loader.exec_module(module) + {mutation} + try: + module.verify_python_runtime({{"python_version": {version!r}}}) + except module.HarnessError as error: + if {message!r} not in str(error): + raise + else: + raise SystemExit("runtime flag was accepted") + """) + checks = ( + (("-I", "-B", "-S"), "sys.dont_write_bytecode = False", + "must use -B"), + (("-B", "-S"), "", "must use -I"), + (("-I", "-B"), "", "must use -S"), + ) + for flags, mutation, message in checks: + script = template.format(path=str(path), mutation=mutation, + version=version, message=message) + result = subprocess.run([sys.executable, *flags, "-c", script], + check=False, capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, + result.stdout + result.stderr) + + def test_harness_authenticates_read_only_sources_before_common_import(self): + source_root = pathlib.Path(common.__file__).resolve().parent + environment = os.environ.copy() + environment.pop("PARQUET_N6_AUTHENTICATED_SNAPSHOT", None) + for harness_name in ("pyarrow.py", "duckdb.py"): + with self.subTest(harness=harness_name), \ + tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory).resolve() + harness = root / harness_name + support = root / "common.py" + harness.write_bytes((source_root / harness_name).read_bytes()) + support.write_bytes(pathlib.Path(common.__file__).read_bytes()) + descriptor = root / "descriptor.toml" + descriptor.write_text( + f'harness_file = "{harness_name}"\n' + f'harness_sha256 = "{common.sha256_file(harness)}"\n' + 'support_files = [{ file = "common.py", sha256 = ' + f'"{common.sha256_file(support)}" }}]\n') + command = [sys.executable, "-I", "-B", "-S", str(harness), + "--repository", str(root), "--descriptor", + str(descriptor), "--help"] + result = subprocess.run(command, check=False, + capture_output=True, text=True, timeout=30, + env=environment) + self.assertEqual(result.returncode, 0, + result.stdout + result.stderr) + self.assertIn("usage:", result.stdout) + marker = root / "common-executed" + with support.open("a") as stream: + stream.write(f"\npathlib.Path({str(marker)!r}).write_text('bad')\n") + result = subprocess.run(command, check=False, + capture_output=True, text=True, timeout=30, + env=environment) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(marker.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/conformance/n6/julia/Manifest.toml b/test/conformance/n6/julia/Manifest.toml new file mode 100644 index 0000000..fea4ce0 --- /dev/null +++ b/test/conformance/n6/julia/Manifest.toml @@ -0,0 +1,170 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.6" +manifest_format = "2.0" +project_hash = "21e5d41e6d5d8aec2a154e411441d1ef11d4e969" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.CRC32]] +deps = ["Zlib_jll"] +git-tree-sha1 = "253002ec391e61dadb453d922f2d278459b0bb0f" +uuid = "b4567568-9dcc-467e-9b62-c342d3a501d3" +version = "1.1.0" + +[[deps.ChunkCodecCore]] +git-tree-sha1 = "1a3ad7e16a321667698a19e77362b35a1e94c544" +uuid = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" +version = "1.0.1" + +[[deps.ChunkCodecLibBrotli]] +deps = ["ChunkCodecCore", "brotli_jll"] +git-tree-sha1 = "45709ad3ba09bdff5e6481d2c1727b1499989997" +uuid = "653b0ff7-85b5-4442-93c1-dcc330d3ec7d" +version = "1.0.0" + +[[deps.ChunkCodecLibLz4]] +deps = ["ChunkCodecCore", "Lz4_jll"] +git-tree-sha1 = "0a4d7695ef98ab714efe5aef26fc35c3b0b4c1ee" +uuid = "7e9cc85e-5614-42a3-ad86-b78f920b38a5" +version = "1.0.0" + +[[deps.ChunkCodecLibSnappy]] +deps = ["ChunkCodecCore", "snappy_jll"] +git-tree-sha1 = "a9e98b8cc7ccdcfcb406773a6c58987daa6eda05" +uuid = "eac87354-86d5-4a5b-ab5f-a6ee56b239b3" +version = "1.0.0" + +[[deps.ChunkCodecLibZlib]] +deps = ["ChunkCodecCore", "Zlib_jll"] +git-tree-sha1 = "d4101e848e8d3f585d61d244c2fe0c80a70e6b3b" +uuid = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" +version = "1.1.0" + +[[deps.ChunkCodecLibZstd]] +deps = ["ChunkCodecCore", "Zstd_jll"] +git-tree-sha1 = "34d9873079e4cb3d0c62926a225136824677073f" +uuid = "55437552-ac27-4d47-9aa3-63184e8fd398" +version = "1.0.0" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.8.0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Lz4_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "191686b1ac1ea9c89fc52e996ad15d1d241d1e33" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.10.1+0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05f45c2e0de6259db764adbfd2f1dc6d3f8de13c" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "2.0.1" + +[[deps.Parquet]] +deps = ["CRC32", "ChunkCodecCore", "ChunkCodecLibBrotli", "ChunkCodecLibLz4", "ChunkCodecLibSnappy", "ChunkCodecLibZlib", "ChunkCodecLibZstd", "DataAPI", "Dates", "Mmap", "Tables", "UUIDs"] +path = "." +uuid = "626c502c-15b0-58ad-a749-f091afb673ae" +version = "1.0.0-DEV" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "0f38a06c83f0007bbab3cf911262841c9a0f07e0" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.13.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[[deps.brotli_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "46fda47f4215c957bc92fd5fbb5ad04fee1e3743" +uuid = "4611771a-a7d2-5e23-8d00-b1becdba1aae" +version = "1.2.0+0" + +[[deps.snappy_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ca88363dd41d2547f52118287dd34dbbc14f3eb7" +uuid = "fe1e1685-f7be-5f59-ac9f-4ca204017dfd" +version = "1.2.3+0" diff --git a/test/conformance/n6/julia/N6ParquetJLHarness.jl b/test/conformance/n6/julia/N6ParquetJLHarness.jl new file mode 100644 index 0000000..c4e3929 --- /dev/null +++ b/test/conformance/n6/julia/N6ParquetJLHarness.jl @@ -0,0 +1,2145 @@ +module N6ParquetJLHarness + +using Dates +using Parquet +using SHA +using TOML + +const N6_ROOT = normpath(joinpath(@__DIR__, "..")) +const REPO_ROOT = normpath(joinpath(N6_ROOT, "..", "..", "..")) +const MODEL_FILE = joinpath(N6_ROOT, "model", "N6StatisticsModel.jl") +const FROZEN_MODEL_SHA256 = + "32c090ed6e0c6af49eabf3f96afc6e17dff630c4e89372367e693202e87c4262" + +function _statidentity(value) + return (value.device, value.inode, value.mode, value.nlink, value.size, + value.mtime, value.ctime) +end + +function _stablefilebytes(path::String, maximum::Int64, label::String) + maximum >= 0 || throw(ArgumentError("$label byte limit is negative")) + maximum < typemax(Int) || throw(ArgumentError("$label byte limit is too large")) + islink(path) && throw(ArgumentError("$label is a symlink")) + before = lstat(path) + isfile(before) || throw(ArgumentError("$label is not a regular file")) + before.size <= maximum || throw(ArgumentError("$label exceeds its byte limit")) + return open(path, "r") do stream + opened = stat(stream) + _statidentity(opened) == _statidentity(before) || throw(ArgumentError( + "$label changed while it was opened")) + bytes = read(stream, Int(maximum) + 1) + length(bytes) <= maximum || throw(ArgumentError( + "$label exceeds its byte limit")) + length(bytes) == opened.size && eof(stream) || throw(ArgumentError( + "$label changed size while it was read")) + final = stat(stream) + current = lstat(path) + isfile(current) && !islink(path) || throw(ArgumentError( + "$label changed type while it was read")) + _statidentity(final) == _statidentity(opened) && + _statidentity(current) == _statidentity(opened) || + throw(ArgumentError("$label changed while it was read")) + return bytes + end +end + +const FROZEN_MODEL_BYTES = _stablefilebytes(MODEL_FILE, Int64(1024 * 1024), + "frozen N6 model source") +bytes2hex(SHA.sha256(FROZEN_MODEL_BYTES)) == FROZEN_MODEL_SHA256 || + error("frozen N6 model source digest differs") +Base.include_string(@__MODULE__, String(copy(FROZEN_MODEL_BYTES)), MODEL_FILE) + +const Model = N6StatisticsModel +const MD = Parquet.Metadata +const TH = Parquet.Thrift +const PRODUCER_DESCRIPTOR_FILE = + joinpath(N6_ROOT, "julia", "parquet-jl-producer.toml") +const PRODUCER_DESCRIPTOR_BYTES = _stablefilebytes(PRODUCER_DESCRIPTOR_FILE, + Int64(4 * 1024 * 1024), "Parquet.jl producer descriptor") +const PRODUCER_DESCRIPTOR_SHA256 = + bytes2hex(SHA.sha256(PRODUCER_DESCRIPTOR_BYTES)) +const PRODUCER_DESCRIPTOR = TOML.parse(String(copy(PRODUCER_DESCRIPTOR_BYTES))) +const FIXTURES_FILE = joinpath(N6_ROOT, "fixtures.toml") +const CAPABILITIES_FILE = joinpath(N6_ROOT, "capabilities.toml") +const MANIFEST_FILE = joinpath(N6_ROOT, "manifest.toml") +const EVIDENCE_SCHEMA_FILE = joinpath(N6_ROOT, "evidence.schema.json") +const CORPUS_MANIFEST_FILE = joinpath(N6_ROOT, "corpus-files.sha256") +const PLAN_FILE = joinpath(REPO_ROOT, "docs", "dev", "n6-statistics-plan.md") +const EVIDENCE_FILE = joinpath(N6_ROOT, "evidence", "parquet-jl.normalized.jsonl") +const CANONICAL_TOOLCHAIN_SHA256 = PRODUCER_DESCRIPTOR_SHA256 +const CANONICAL_WRITER_EXECUTABLE_SHA256 = + PRODUCER_DESCRIPTOR["julia_executable_sha256"] +const CANONICAL_SOURCE_REVISION = + PRODUCER_DESCRIPTOR["source_composite_sha256"] +const CANONICAL_WRITER_VERSION = v"1.12.6" +const MAX_DECLARATION_BYTES = Int64(1024 * 1024) +const MAX_GENERATED_BYTES = Int64(32 * 1024 * 1024) +const MAX_ROWS = 64 +const GENERATED_CASE_COUNT = 12 +const EXPECTED_PROFILES = Dict( + "julia-writer-type-order" => ("writer-type-order-v1", Int64(6001), "type-order"), + "julia-writer-ieee-order" => ("writer-ieee-order-v1", Int64(6002), "ieee-order"), + "julia-writer-undefined-order" => ("writer-undefined-order-v1", Int64(6003), "undefined-order"), + "julia-writer-statistics-disabled" => ("writer-statistics-disabled-v1", Int64(6004), "statistics-disabled"), + "julia-writer-oversized-bounds" => ("writer-oversized-bounds-v1", Int64(6005), "one-byte-over"), + "julia-writer-nested-row-groups" => ("writer-nested-row-groups-v1", Int64(6006), "nested-row-groups"), + "julia-reader-untrusted-producer" => ("reader-untrusted-producer-v1", Int64(6007), "parquet-mr-1.7.0"), + "julia-reader-no-pruning-absent" => ("reader-no-pruning-v1", Int64(6008), "absent"), + "julia-reader-no-pruning-trusted" => ("reader-no-pruning-v1", Int64(6008), "trusted"), + "julia-reader-no-pruning-untrusted" => ("reader-no-pruning-v1", Int64(6008), "producer-untrusted"), + "julia-reader-no-pruning-oversized" => ("reader-no-pruning-v1", Int64(6008), "oversized"), + "julia-reader-no-pruning-invalid" => ("reader-no-pruning-v1", Int64(6008), "semantically-unusable"), +) + +mutable struct SeedStream + state::UInt64 +end + +function SeedStream(seed::Integer) + seed >= 0 || throw(ArgumentError("generator seed must be nonnegative")) + return SeedStream(xor(UInt64(seed), 0x9e3779b97f4a7c15)) +end + +function nextseed!(stream::SeedStream) + stream.state = stream.state * 0x5851f42d4c957f2d + 0x14057b7ef767814f + return stream.state +end + +function seedbyte!(stream::SeedStream) + return UInt8(nextseed!(stream) >> 56) +end + +struct ProfileOutput + table::Any + bytes::Vector{UInt8} + statistics_enabled::Bool + statistics_limit::Int64 + comparison_bytes::Union{Nothing,Vector{UInt8}} +end + +struct CheckedCase + declaration::Dict{String,Any} + bytes::Vector{UInt8} + file_record::Dict{String,Any} + column_records::Vector{Dict{String,Any}} + logical_values_sha256::String + metadata_sha256::String + assertion_facts::Dict{String,Any} + no_pruning_sha256::Union{Nothing,String} +end + +struct HarnessOutput + files::Vector{Pair{String,Vector{UInt8}}} + evidence::Vector{UInt8} + checked::Vector{CheckedCase} +end + +mutable struct TraceSource <: Parquet.AbstractSource + bytes::Vector{UInt8} + reads::Vector{Tuple{Int64,Int64}} +end + +function Parquet.sourcelength(source::TraceSource) + return Int64(length(source.bytes)) +end + +function Parquet.readrange(source::TraceSource, offset::Integer, count::Integer) + offset64 = Int64(offset) + count64 = Int64(count) + offset64 >= 0 || throw(ArgumentError("trace source offset is negative")) + count64 >= 0 || throw(ArgumentError("trace source count is negative")) + stop = Base.checked_add(offset64, count64) + stop <= length(source.bytes) || throw(BoundsError(source.bytes, + (offset64, count64))) + push!(source.reads, (offset64, count64)) + first = Int(offset64) + 1 + return @view source.bytes[first:(first + Int(count64) - 1)] +end + +function Parquet.close!(::TraceSource) + return +end + +function filehash(path::AbstractString; maximum::Int64=MAX_DECLARATION_BYTES, + label::String="hashed input") + return bytehash(_stablefilebytes(String(path), maximum, label)) +end + +function bytehash(bytes::AbstractVector{UInt8}) + return bytes2hex(SHA.sha256(bytes)) +end + +function _hex4(value::UInt32) + return lowercase(string(value; base=16, pad=8)) +end + +function _hex8(value::UInt64) + return lowercase(string(value; base=16, pad=16)) +end + +function _jsonstring!(output::IO, value::AbstractString) + write(output, UInt8('"')) + for character in value + character == '"' && (write(output, "\\\""); continue) + character == '\\' && (write(output, "\\\\"); continue) + character == '\b' && (write(output, "\\b"); continue) + character == '\f' && (write(output, "\\f"); continue) + character == '\n' && (write(output, "\\n"); continue) + character == '\r' && (write(output, "\\r"); continue) + character == '\t' && (write(output, "\\t"); continue) + if Int(character) < 0x20 + write(output, "\\u", lowercase(string(Int(character); base=16, pad=4))) + else + write(output, string(character)) + end + end + write(output, UInt8('"')) + return +end + +function _canonicaljson!(output::IO, value) + value === nothing && (write(output, "null"); return) + value isa Bool && (write(output, value ? "true" : "false"); return) + value isa Integer && !(value isa Bool) && (write(output, string(value)); return) + value isa AbstractFloat && throw(ArgumentError( + "floating JSON numbers are forbidden in N6 evidence")) + value isa AbstractString && (_jsonstring!(output, value); return) + if value isa AbstractDict + keys_ = sort!(String[String(key) for key in keys(value)]) + length(keys_) == length(unique(keys_)) || throw(ArgumentError( + "canonical JSON object has duplicate string keys")) + write(output, UInt8('{')) + for (index, key) in enumerate(keys_) + index == 1 || write(output, UInt8(',')) + _jsonstring!(output, key) + write(output, UInt8(':')) + _canonicaljson!(output, value[key]) + end + write(output, UInt8('}')) + return + end + if value isa Union{Tuple,AbstractVector} + write(output, UInt8('[')) + for (index, item) in enumerate(value) + index == 1 || write(output, UInt8(',')) + _canonicaljson!(output, item) + end + write(output, UInt8(']')) + return + end + throw(ArgumentError("unsupported canonical JSON value $(typeof(value))")) +end + +function canonicaljson(value) + output = IOBuffer() + _canonicaljson!(output, value) + return String(take!(output)) +end + +function canonicalhash(value) + return bytehash(codeunits(canonicaljson(value))) +end + +function _typedinteger(value::Integer) + return Dict{String,Any}( + "type" => string(nameof(typeof(value))), + "value" => string(value), + ) +end + +function _logicalvalue(value) + value === missing && return nothing + value isa Bool && return value + value isa Integer && return _typedinteger(value) + value isa Float16 && return Dict{String,Any}( + "bits" => lowercase(string(reinterpret(UInt16, value); base=16, pad=4)), + "type" => "Float16") + value isa Float32 && return Dict{String,Any}( + "bits" => _hex4(reinterpret(UInt32, value)), "type" => "Float32") + value isa Float64 && return Dict{String,Any}( + "bits" => _hex8(reinterpret(UInt64, value)), "type" => "Float64") + value isa AbstractString && return String(value) + value isa Parquet.JSONValue && return Dict{String,Any}( + "json_hex" => bytes2hex(value.bytes)) + value isa Parquet.BSONValue && return Dict{String,Any}( + "bson_hex" => bytes2hex(value.bytes)) + value isa Parquet.Interval && return Dict{String,Any}( + "days" => string(value.days), "milliseconds" => string(value.milliseconds), + "months" => string(value.months)) + value isa Parquet.Decimal && return Dict{String,Any}( + "scale" => string(value.scale), "unscaled" => string(value.unscaled)) + value isa Date && return Dict{String,Any}( + "date_days" => string(Dates.value(value))) + value isa DateTime && return Dict{String,Any}( + "datetime_millis" => string(Dates.value(value))) + value isa Time && return Dict{String,Any}( + "time_nanos" => string(Dates.value(value))) + value isa Parquet.Timestamp && return Dict{String,Any}( + "adjusted" => value.is_adjusted_to_utc, + "ticks" => string(value.ticks), "type" => string(typeof(value))) + value isa Parquet.StructValue && return Dict{String,Any}( + String(value.names[index]) => _logicalvalue(value[index]) + for index in 1:length(value)) + value isa NamedTuple && return Dict{String,Any}( + String(name) => _logicalvalue(getproperty(value, name)) for name in keys(value)) + value isa Tuple && all(item -> item isa UInt8, value) && + return Dict{String,Any}("bytes_hex" => bytes2hex(UInt8[value...])) + value isa Tuple && return Any[_logicalvalue(item) for item in value] + value isa Pair && return Dict{String,Any}( + "key" => _logicalvalue(first(value)), "value" => _logicalvalue(last(value))) + value isa AbstractVector{UInt8} && return Dict{String,Any}( + "bytes_hex" => bytes2hex(value)) + value isa AbstractVector && return Any[_logicalvalue(item) for item in value] + value isa AbstractDict && return Any[_logicalvalue(pair) for pair in value] + throw(ArgumentError("unsupported logical value $(typeof(value))")) +end + +function logicalrows(columns::NamedTuple) + names = keys(columns) + values = Base.values(columns) + rows = isempty(values) ? 0 : length(first(values)) + all(column -> length(column) == rows, values) || throw(ArgumentError( + "logical columns have different lengths")) + output = [] + sizehint!(output, rows) + for row in 1:rows + push!(output, Dict{String,Any}( + String(name) => _logicalvalue(column[row]) + for (name, column) in zip(names, values))) + end + return output +end + +function checkeddeclarations(fixtures::Dict{String,Any}) + declarations = fixtures["generated_case"] + length(declarations) == GENERATED_CASE_COUNT || throw(ArgumentError( + "N6 harness requires exactly $GENERATED_CASE_COUNT generated cases")) + ids = String[declaration["id"] for declaration in declarations] + Set(ids) == Set(keys(EXPECTED_PROFILES)) || throw(ArgumentError( + "generated case IDs differ from the reviewed profile set")) + for declaration in declarations + id = declaration["id"] + declaration["output_file"] == "generated/$id.parquet" || + throw(ArgumentError("generated output path differs for $id")) + expected = EXPECTED_PROFILES[id] + actual = (declaration["generator_profile"], declaration["generator_seed"], + declaration["variant_id"]) + actual == expected || throw(ArgumentError( + "generated identity differs for $id")) + declaration["output_identity_status"] in ("planned", "verified") || + throw(ArgumentError("unsupported output identity status for $id")) + declaration["authority"] == "parquet-jl" || throw(ArgumentError( + "generated case authority differs for $id")) + declaration["row_group_count"] >= 0 || throw(ArgumentError( + "negative row-group count for $id")) + declaration["leaf_count"] >= 0 || throw(ArgumentError( + "negative leaf count for $id")) + end + outputs = String[declaration["output_file"] for declaration in declarations] + length(unique(outputs)) == length(outputs) || throw(ArgumentError( + "generated case output paths are not unique")) + return fixtures, declarations +end + +function _writerbytes(table; statistics::Bool=true, limit::Int64=4096, + pageversion::Symbol=:v2, rowgroupsize::Int=3) + rows = isempty(values(table)) ? 0 : length(first(values(table))) + rows <= MAX_ROWS || throw(ArgumentError("generator row count exceeds $MAX_ROWS")) + limits = Parquet.Limits(max_statistics_value_bytes=limit, + max_materialized_bytes=256 * 1024 * 1024, + max_string_bytes=16 * 1024 * 1024, + max_container_elements=100_000) + bytes = Parquet._encodefile(table; checksum=false, dictionary=false, + codec=:uncompressed, pageversion=pageversion, encoding=:plain, + rowgroupsize=rowgroupsize, pagesize=nothing, pageindex=true, + statistics=statistics, limits=limits) + length(bytes) <= MAX_GENERATED_BYTES || throw(ArgumentError( + "generated Parquet output exceeds $MAX_GENERATED_BYTES bytes")) + return bytes +end + +function _typeordertable(seed::Int64) + stream = SeedStream(seed) + marker = Int32(seedbyte!(stream)) + signed = Union{Missing,Int32}[-1000 - marker, missing, 9, 400, -7, 1200 + marker] + unsigned = UInt32[0, 0x80000000, typemax(UInt32), UInt32(seedbyte!(stream)), 1, 9] + raw = Union{Missing,Vector{UInt8}}[ + UInt8[0x00, seedbyte!(stream)], missing, UInt8[0xff], UInt8[], + UInt8[0x7f, 0x00], UInt8[0x80], + ] + text = Union{Missing,String}["z", missing, "a", "aa", "a\0b", "zz"] + decimals = Union{Missing,Parquet.Decimal}[ + Parquet.Decimal(-90001, 2), missing, Parquet.Decimal(0, 2), + Parquet.Decimal(12345, 2), Parquet.Decimal(-1, 2), + Parquet.Decimal(99999, 2), + ] + decimal = Parquet.LogicalColumn(decimals, :decimal; precision=9, scale=2) + dates = Union{Missing,Date}[ + Date(1900, 1, 1), missing, Date(1970, 1, 1), Date(2000, 2, 29), + Date(1969, 12, 31), Date(2100, 1, 1), + ] + flag = Union{Missing,Bool}[false, missing, true, true, false, true] + fixed = NTuple{3,UInt8}[(0x00, 0xff, seedbyte!(stream)), (0x10, 0x00, 0x00), + (0xff, 0x00, 0x00), (0x01, 0x02, 0x03), (0x7f, 0xff, 0xff), + (0x80, 0x00, 0x00)] + return (; signed, unsigned, raw, text, decimal, dates, flag, fixed) +end + +function _ieeetable(seed::Int64) + stream = SeedStream(seed) + payload16 = UInt16(seedbyte!(stream) & 0x3f) | 0x0001 + payload32 = UInt32(seedbyte!(stream)) | UInt32(1) + payload64 = UInt64(seedbyte!(stream)) | UInt64(1) + half = Union{Missing,Float16}[ + missing, + reinterpret(Float16, UInt16(0x8000)), + reinterpret(Float16, UInt16(0x0000)), + reinterpret(Float16, UInt16(0xbc00)), + reinterpret(Float16, UInt16(0x7c00)), + reinterpret(Float16, UInt16(0x7e00 | payload16)), + reinterpret(Float16, UInt16(0xfc01)), + reinterpret(Float16, UInt16(0x7c01)), + reinterpret(Float16, UInt16(0x7e10)), + reinterpret(Float16, UInt16(0xfe20)), + reinterpret(Float16, UInt16(0x7d55)), + reinterpret(Float16, UInt16(0xfd23)), + ] + single = Union{Missing,Float32}[ + missing, + reinterpret(Float32, UInt32(0x80000000)), + reinterpret(Float32, UInt32(0x00000000)), + reinterpret(Float32, UInt32(0xbf800000)), + reinterpret(Float32, UInt32(0x7f800000)), + reinterpret(Float32, UInt32(0x7fc00000 | payload32)), + reinterpret(Float32, UInt32(0xff800001)), + reinterpret(Float32, UInt32(0x7f800001)), + reinterpret(Float32, UInt32(0x7fc12345)), + reinterpret(Float32, UInt32(0xffc54321)), + reinterpret(Float32, UInt32(0x7fa00011)), + reinterpret(Float32, UInt32(0x7fe22222)), + ] + double = Union{Missing,Float64}[ + missing, + reinterpret(Float64, UInt64(0x8000000000000000)), + reinterpret(Float64, UInt64(0x0000000000000000)), + reinterpret(Float64, UInt64(0xbff0000000000000)), + reinterpret(Float64, UInt64(0x7ff0000000000000)), + reinterpret(Float64, UInt64(0x7ff8000000000000) | payload64), + reinterpret(Float64, UInt64(0xfff0000000000001)), + reinterpret(Float64, UInt64(0x7ff0000000000001)), + reinterpret(Float64, UInt64(0x7ff8000000012345)), + reinterpret(Float64, UInt64(0xfff8000000000022)), + reinterpret(Float64, UInt64(0x7ff4000000000033)), + reinterpret(Float64, UInt64(0x7ff2000000000044)), + ] + return (; half, single, double) +end + +function _undefinedtable(seed::Int64) + stream = SeedStream(seed) + marker = UInt32(seedbyte!(stream)) + first = Parquet.Interval[ + Parquet.Interval(0, 0, marker), Parquet.Interval(1, 2, 3), + Parquet.Interval(9, 8, 7), Parquet.Interval(0, 1, 0), + ] + second = Union{Missing,Parquet.Interval}[ + missing, Parquet.Interval(marker, 0, 1), missing, Parquet.Interval(4, 5, 6), + ] + unknown = Missing[missing, missing, missing, missing] + return (; first, second, unknown) +end + +function _disabledtable(seed::Int64) + stream = SeedStream(seed) + number = Union{Missing,Int32}[ + Int32(seedbyte!(stream)), missing, -3, 7, 0, 99] + text = Union{Missing,String}["left", missing, "right", "", "middle", "tail"] + return (; number, text) +end + +function _oversizedtable(seed::Int64) + stream = SeedStream(seed) + low = String(vcat(UInt8['a'], fill(UInt8('x'), 4095), seedbyte!(stream) & 0x0f | 0x30)) + high = String(vcat(UInt8['z'], fill(UInt8('y'), 4095), seedbyte!(stream) & 0x0f | 0x40)) + return (value=String[low, high, low],) +end + +function _nestedtable(seed::Int64) + stream = SeedStream(seed) + element = Union{Missing,Int32} + rowtype = NamedTuple{(:id,:items),Tuple{Int32,Union{Missing,Vector{element}}}} + rows = Union{Missing,rowtype}[ + missing, + rowtype((Int32(seedbyte!(stream)), missing)), + rowtype((Int32(2), element[])), + rowtype((Int32(3), element[missing, -1, 8])), + rowtype((Int32(4), element[9])), + rowtype((Int32(5), element[missing, 10])), + ] + tag = Union{Missing,String}[missing, "a", "b", "c", missing, "z"] + flag = Bool[false, true, false, true, true, false] + return (; tag, rows, flag) +end + +function _untrustedtable(seed::Int64) + stream = SeedStream(seed) + binary = Union{Missing,String}[ + "z", missing, "a", "middle", "tail", string("seed-", seedbyte!(stream))] + signed = Union{Missing,Int32}[-10, missing, 9, 100, -99, Int32(seedbyte!(stream))] + return (; binary, signed) +end + +function _nopruningtable(seed::Int64) + stream = SeedStream(seed) + marker = Int(seedbyte!(stream)) + json = Union{Missing,Parquet.JSONValue}[ + Parquet.JSONValue(codeunits("{\"value\":$marker}")), + missing, + Parquet.JSONValue(codeunits("[1,2,3]")), + Parquet.JSONValue(codeunits("{\"nested\":true}")), + Parquet.JSONValue(codeunits("null")), + Parquet.JSONValue(codeunits("\"tail\"")), + ] + return (; json) +end + +function _columnmetadata(metadata::MD.ColumnMetaData, + statistics::Union{Nothing,MD.Statistics}) + return MD.ColumnMetaData( + type_=metadata.type_, encodings=metadata.encodings, + path_in_schema=metadata.path_in_schema, codec=metadata.codec, + num_values=metadata.num_values, + total_uncompressed_size=metadata.total_uncompressed_size, + total_compressed_size=metadata.total_compressed_size, + key_value_metadata=metadata.key_value_metadata, + data_page_offset=metadata.data_page_offset, + index_page_offset=metadata.index_page_offset, + dictionary_page_offset=metadata.dictionary_page_offset, + statistics=statistics, encoding_stats=metadata.encoding_stats, + bloom_filter_offset=metadata.bloom_filter_offset, + bloom_filter_length=metadata.bloom_filter_length, + size_statistics=metadata.size_statistics, + geospatial_statistics=metadata.geospatial_statistics, + unknown_fields=metadata.unknown_fields) +end + +function _columnchunk(chunk::MD.ColumnChunk, metadata::MD.ColumnMetaData) + return MD.ColumnChunk( + file_path=chunk.file_path, file_offset=chunk.file_offset, + meta_data=metadata, offset_index_offset=chunk.offset_index_offset, + offset_index_length=chunk.offset_index_length, + column_index_offset=chunk.column_index_offset, + column_index_length=chunk.column_index_length, + crypto_metadata=chunk.crypto_metadata, + encrypted_column_metadata=chunk.encrypted_column_metadata, + unknown_fields=chunk.unknown_fields) +end + +function _rowgroup(group::MD.RowGroup, columns::Vector{MD.ColumnChunk}) + return MD.RowGroup(columns=columns, total_byte_size=group.total_byte_size, + num_rows=group.num_rows, sorting_columns=group.sorting_columns, + file_offset=group.file_offset, + total_compressed_size=group.total_compressed_size, + ordinal=group.ordinal, unknown_fields=group.unknown_fields) +end + +function _filemetadata(metadata::MD.FileMetaData, + rowgroups::Vector{MD.RowGroup}, createdby::Union{Nothing,String}, + orders::Union{Nothing,Vector{MD.ColumnOrder}}) + return MD.FileMetaData(version=metadata.version, schema=metadata.schema, + num_rows=metadata.num_rows, row_groups=rowgroups, + key_value_metadata=metadata.key_value_metadata, created_by=createdby, + column_orders=orders, + encryption_algorithm=metadata.encryption_algorithm, + footer_signing_key_metadata=metadata.footer_signing_key_metadata, + unknown_fields=metadata.unknown_fields) +end + +function _decodefooter(bytes::Vector{UInt8}) + file = Parquet.File(bytes) + try + metadata = TH.decode(copy(file.footer.bytes), MD.FileMetaData) + return metadata, Int(file.footer.offset), Int(file.footer.length) + finally + close(file) + end +end + +function _replacefooter(bytes::Vector{UInt8}, metadata::MD.FileMetaData) + _, offset, _ = _decodefooter(bytes) + prefix = copy(@view bytes[1:offset]) + footer = TH.encode(metadata) + length(footer) <= typemax(UInt32) || throw(ArgumentError( + "generated footer exceeds UInt32")) + output = vcat(prefix, footer) + Parquet._writelittle!(output, UInt32(length(footer))) + append!(output, Parquet.PARQUET_MAGIC) + length(output) <= MAX_GENERATED_BYTES || throw(ArgumentError( + "mutated output exceeds $MAX_GENERATED_BYTES bytes")) + return output +end + +function _mapstatistics(transform::Function, metadata::MD.FileMetaData) + groups = MD.RowGroup[] + sizehint!(groups, length(metadata.row_groups)) + for (groupindex, group) in enumerate(metadata.row_groups) + columns = MD.ColumnChunk[] + sizehint!(columns, length(group.columns)) + for (leafindex, chunk) in enumerate(group.columns) + column = something(chunk.meta_data) + statistics = transform(column.statistics, groupindex, leafindex) + push!(columns, _columnchunk(chunk, + _columnmetadata(column, statistics))) + end + push!(groups, _rowgroup(group, columns)) + end + return groups +end + +function _deprecatedstatistics(statistics::Union{Nothing,MD.Statistics}) + statistics === nothing && return nothing + return MD.Statistics(max=statistics.max_value, min=statistics.min_value, + null_count=statistics.null_count, + distinct_count=statistics.distinct_count, + nan_count=statistics.nan_count, + unknown_fields=statistics.unknown_fields) +end + +function _untrustedmutation(bytes::Vector{UInt8}, declaration) + metadata, _, _ = _decodefooter(bytes) + groups = _mapstatistics(metadata) do statistics, _, leafindex + return leafindex == 2 ? _deprecatedstatistics(statistics) : statistics + end + createdby = declaration["mutation"]["created_by"] + updated = _filemetadata(metadata, groups, createdby, metadata.column_orders) + return _replacefooter(bytes, updated) +end + +function _nopruneabsent(metadata::MD.FileMetaData) + groups = _mapstatistics(metadata) do _, _, _ + return nothing + end + return _filemetadata(metadata, groups, metadata.created_by, nothing) +end + +function _nopruneoversized(metadata::MD.FileMetaData, bytes::Int64) + raw = fill(UInt8('x'), Int(bytes)) + groups = _mapstatistics(metadata) do statistics, _, _ + statistics === nothing && throw(AssertionError( + "no-pruning base statistics are absent")) + return MD.Statistics(null_count=statistics.null_count, + distinct_count=statistics.distinct_count, + min_value=copy(raw), max_value=copy(raw), + is_min_value_exact=true, is_max_value_exact=true, + nan_count=statistics.nan_count, + unknown_fields=statistics.unknown_fields) + end + return _filemetadata(metadata, groups, metadata.created_by, + metadata.column_orders) +end + +function _nopruneinvalid(metadata::MD.FileMetaData, raw::Vector{UInt8}) + groups = _mapstatistics(metadata) do statistics, _, _ + statistics === nothing && throw(AssertionError( + "no-pruning base statistics are absent")) + return MD.Statistics(null_count=statistics.null_count, + distinct_count=statistics.distinct_count, + min_value=copy(raw), max_value=statistics.max_value, + is_min_value_exact=true, + is_max_value_exact=statistics.is_max_value_exact, + nan_count=statistics.nan_count, + unknown_fields=statistics.unknown_fields) + end + return _filemetadata(metadata, groups, metadata.created_by, + metadata.column_orders) +end + +function _nopruningmutation(bytes::Vector{UInt8}, declaration) + metadata, _, _ = _decodefooter(bytes) + mutation = declaration["mutation"] + state = mutation["statistics_state"] + updated = if state == "absent" + _nopruneabsent(metadata) + elseif state in ("trusted", "producer-untrusted") + _filemetadata(metadata, metadata.row_groups, mutation["created_by"], + metadata.column_orders) + elseif state == "oversized" + _nopruneoversized(metadata, mutation["bound_bytes"]) + elseif state == "semantically-unusable" + mutation["field"] == "min_value" || throw(ArgumentError( + "unsupported no-pruning invalid field")) + _nopruneinvalid(metadata, hex2bytes(mutation["value_hex"])) + else + throw(ArgumentError("unsupported no-pruning state $state")) + end + return _replacefooter(bytes, updated) +end + +function generateprofile(declaration::Dict{String,Any}) + profile = declaration["generator_profile"] + seed = declaration["generator_seed"] + if profile == "writer-type-order-v1" + table = _typeordertable(seed) + return ProfileOutput(table, _writerbytes(table; pageversion=:v1), true, + Int64(4096), nothing) + elseif profile == "writer-ieee-order-v1" + table = _ieeetable(seed) + return ProfileOutput(table, _writerbytes(table; pageversion=:v2, + rowgroupsize=6), true, + Int64(4096), nothing) + elseif profile == "writer-undefined-order-v1" + table = _undefinedtable(seed) + return ProfileOutput(table, _writerbytes(table; pageversion=:v1, + rowgroupsize=4), true, Int64(4096), nothing) + elseif profile == "writer-statistics-disabled-v1" + table = _disabledtable(seed) + enabled = _writerbytes(table; statistics=true, pageversion=:v2) + disabled = _writerbytes(table; statistics=false, pageversion=:v2) + return ProfileOutput(table, disabled, false, Int64(4096), enabled) + elseif profile == "writer-oversized-bounds-v1" + table = _oversizedtable(seed) + return ProfileOutput(table, _writerbytes(table; limit=4096, + pageversion=:v2, rowgroupsize=3), true, Int64(4096), nothing) + elseif profile == "writer-nested-row-groups-v1" + table = _nestedtable(seed) + return ProfileOutput(table, _writerbytes(table; pageversion=:v2), true, + Int64(4096), nothing) + elseif profile == "reader-untrusted-producer-v1" + table = _untrustedtable(seed) + base = _writerbytes(table; pageversion=:v1) + return ProfileOutput(table, _untrustedmutation(base, declaration), true, + Int64(4096), nothing) + elseif profile == "reader-no-pruning-v1" + table = _nopruningtable(seed) + base = _writerbytes(table; pageversion=:v2) + return ProfileOutput(table, _nopruningmutation(base, declaration), true, + Int64(4096), nothing) + end + throw(ArgumentError("unsupported generator profile $profile")) +end + +function _require(condition::Bool, message::AbstractString) + condition || throw(AssertionError(String(message))) + return +end + +function _enumname(value) + for (candidate, name) in TH.enumnames(typeof(value)) + candidate == value.value && return String(name) + end + throw(ArgumentError("unknown $(typeof(value)) value $(value.value)")) +end + +function _logicalname(element::MD.SchemaElement) + kind = Parquet._logicalkind(element) + kind === nothing && return "NONE" + kind isa Parquet._IntegerLogicalKind && return "INTEGER" + kind isa Parquet._TimeLogicalKind && return "TIME" + kind isa Parquet._TimestampLogicalKind && return "TIMESTAMP" + kind === :string && return "STRING" + kind === :enum && return "ENUM" + kind === :decimal && return "DECIMAL" + kind === :date && return "DATE" + kind === :unknown && return "UNKNOWN" + kind === :json && return "JSON" + kind === :bson && return "BSON" + kind === :uuid && return "UUID" + kind === :float16 && return "FLOAT16" + kind === :interval && return "INTERVAL" + return uppercase(String(kind)) +end + +function _timeunitname(unit::UInt8) + unit == Parquet._TEMPORAL_MILLIS && return "MILLIS" + unit == Parquet._TEMPORAL_MICROS && return "MICROS" + unit == Parquet._TEMPORAL_NANOS && return "NANOS" + throw(ArgumentError("unknown temporal unit $unit")) +end + +function _geospatialparameters(element::MD.SchemaElement) + logical = element.logicalType + logical === nothing && return nothing, nothing + if logical.GEOMETRY !== nothing + return logical.GEOMETRY.crs, nothing + elseif logical.GEOGRAPHY !== nothing + algorithm = logical.GEOGRAPHY.algorithm + return logical.GEOGRAPHY.crs, + algorithm === nothing ? nothing : _enumname(algorithm) + end + return nothing, nothing +end + +function normalizeleaf(element::MD.SchemaElement) + kind = Parquet._logicalkind(element) + logical = _logicalname(element) + precision = nothing + scale = nothing + bitwidth = nothing + signed = nothing + timeunit = nothing + adjusted = nothing + if kind === :decimal + decimal = Parquet._decimalparameters(element) + precision = Int(first(decimal)) + scale = Int(last(decimal)) + elseif kind isa Parquet._IntegerLogicalKind + bitwidth = Int(kind.bitwidth) + signed = kind.signed + elseif kind isa Union{Parquet._TimeLogicalKind,Parquet._TimestampLogicalKind} + timeunit = _timeunitname(kind.unit) + adjusted = kind.is_adjusted_to_utc + end + crs, algorithm = _geospatialparameters(element) + return Dict{String,Any}( + "physical_type" => _enumname(element.type_), + "logical_type" => logical, + "converted_type" => element.converted_type === nothing ? nothing : + _enumname(element.converted_type), + "type_length" => element.type_length === nothing ? nothing : + Int(element.type_length), + "precision" => precision, + "scale" => scale, + "bit_width" => bitwidth, + "is_signed" => signed, + "time_unit" => timeunit, + "is_adjusted_to_utc" => adjusted, + "crs" => crs, + "geography_algorithm" => algorithm, + ) +end + +function normalizeorder(order::Union{Nothing,MD.ColumnOrder}) + order === nothing && return Dict{String,Any}( + "state" => "ABSENT", "field_id" => nothing, + "wire_type" => nothing, "header_hex" => nothing) + encoded = TH.encode(order) + if order.TYPE_ORDER !== nothing + _require(encoded == UInt8[0x1c, 0x00, 0x00], + "TYPE_ORDER does not use the canonical Compact-Thrift bytes") + return Dict{String,Any}( + "state" => "TYPE_ORDER", "field_id" => 1, + "wire_type" => 12, "header_hex" => "1c") + elseif order.IEEE_754_TOTAL_ORDER !== nothing + _require(encoded == UInt8[0x2c, 0x00, 0x00], + "IEEE_754_TOTAL_ORDER does not use the canonical Compact-Thrift bytes") + return Dict{String,Any}( + "state" => "IEEE_754_TOTAL_ORDER", "field_id" => 2, + "wire_type" => 12, "header_hex" => "2c") + end + throw(AssertionError("generated ColumnOrder has no reviewed union member")) +end + +function _nullableinteger(value::Union{Nothing,Integer}) + return value === nothing ? nothing : string(Int64(value)) +end + +function _nullablehex(value::Union{Nothing,AbstractVector{UInt8}}) + return value === nothing ? nothing : bytes2hex(value) +end + +function normalizecolumn(caseid::String, relative::String, rowgroup::Int, + leafindex::Int, node::Parquet.SchemaNode, column::MD.ColumnMetaData, + order::Union{Nothing,MD.ColumnOrder}) + statistics = column.statistics + unknown = statistics === nothing ? Int[] : + sort!(unique!(Int[field.id for field in statistics.unknown_fields])) + return Dict{String,Any}( + "record" => "column_statistics", + "schema_version" => 2, + "case_id" => caseid, + "file" => relative, + "row_group" => rowgroup - 1, + "leaf" => leafindex - 1, + "path" => copy(node.path), + "leaf_schema" => normalizeleaf(node.element), + "column_order" => normalizeorder(order), + "num_values" => string(column.num_values), + "has_statistics" => statistics !== nothing, + "deprecated_min_hex" => statistics === nothing ? nothing : + _nullablehex(statistics.min), + "deprecated_max_hex" => statistics === nothing ? nothing : + _nullablehex(statistics.max), + "min_value_hex" => statistics === nothing ? nothing : + _nullablehex(statistics.min_value), + "max_value_hex" => statistics === nothing ? nothing : + _nullablehex(statistics.max_value), + "is_min_value_exact" => statistics === nothing ? nothing : + statistics.is_min_value_exact, + "is_max_value_exact" => statistics === nothing ? nothing : + statistics.is_max_value_exact, + "null_count" => statistics === nothing ? nothing : + _nullableinteger(statistics.null_count), + "distinct_count" => statistics === nothing ? nothing : + _nullableinteger(statistics.distinct_count), + "nan_count" => statistics === nothing ? nothing : + _nullableinteger(statistics.nan_count), + "unknown_statistics_field_ids" => unknown, + ) +end + +const MODEL_PHYSICAL_TYPES = Dict( + "BOOLEAN" => Model.PHYSICAL_BOOLEAN, + "INT32" => Model.PHYSICAL_INT32, + "INT64" => Model.PHYSICAL_INT64, + "INT96" => Model.PHYSICAL_INT96, + "FLOAT" => Model.PHYSICAL_FLOAT, + "DOUBLE" => Model.PHYSICAL_DOUBLE, + "BYTE_ARRAY" => Model.PHYSICAL_BYTE_ARRAY, + "FIXED_LEN_BYTE_ARRAY" => Model.PHYSICAL_FIXED_LEN_BYTE_ARRAY, +) + +const MODEL_LOGICAL_TYPES = Dict( + "NONE" => Model.LOGICAL_NONE, + "STRING" => Model.LOGICAL_STRING, + "ENUM" => Model.LOGICAL_ENUM, + "JSON" => Model.LOGICAL_JSON, + "BSON" => Model.LOGICAL_BSON, + "UUID" => Model.LOGICAL_UUID, + "DECIMAL" => Model.LOGICAL_DECIMAL, + "DATE" => Model.LOGICAL_DATE, + "TIME" => Model.LOGICAL_TIME, + "TIMESTAMP" => Model.LOGICAL_TIMESTAMP, + "FLOAT16" => Model.LOGICAL_FLOAT16, + "INTERVAL" => Model.LOGICAL_INTERVAL, + "UNKNOWN" => Model.LOGICAL_UNKNOWN, + "VARIANT" => Model.LOGICAL_VARIANT, + "GEOMETRY" => Model.LOGICAL_GEOMETRY, + "GEOGRAPHY" => Model.LOGICAL_GEOGRAPHY, + "LIST" => Model.LOGICAL_LIST, + "MAP" => Model.LOGICAL_MAP, +) + +function modelspec(leaf::Dict{String,Any}) + logicalname = leaf["logical_type"] + logical = if logicalname == "INTEGER" + leaf["is_signed"] ? Model.LOGICAL_SIGNED_INTEGER : + Model.LOGICAL_UNSIGNED_INTEGER + else + MODEL_LOGICAL_TYPES[logicalname] + end + timeunit = leaf["time_unit"] == "MILLIS" ? Model.TIME_MILLIS : + leaf["time_unit"] == "MICROS" ? Model.TIME_MICROS : + leaf["time_unit"] == "NANOS" ? Model.TIME_NANOS : nothing + return Model.LeafSpec(MODEL_PHYSICAL_TYPES[leaf["physical_type"]]; + logical=logical, type_length=leaf["type_length"], + bit_width=leaf["bit_width"], precision=leaf["precision"], + time_unit=timeunit) +end + +function modelstatistics(statistics::Union{Nothing,MD.Statistics}) + statistics === nothing && return Model.RawStatistics() + return Model.RawStatistics( + modern_lower=statistics.min_value === nothing ? nothing : + copy(statistics.min_value), + modern_upper=statistics.max_value === nothing ? nothing : + copy(statistics.max_value), + deprecated_lower=statistics.min === nothing ? nothing : + copy(statistics.min), + deprecated_upper=statistics.max === nothing ? nothing : + copy(statistics.max), + null_count=statistics.null_count, + nan_count=statistics.nan_count, + distinct_count=statistics.distinct_count, + lower_exact=statistics.is_min_value_exact, + upper_exact=statistics.is_max_value_exact, + ) +end + +function modelorders(orders::Union{Nothing,Vector{MD.ColumnOrder}}) + orders === nothing && return nothing + output = Model.DeclaredOrder[] + sizehint!(output, length(orders)) + for order in orders + if order.TYPE_ORDER !== nothing + push!(output, Model.ORDER_TYPE) + elseif order.IEEE_754_TOTAL_ORDER !== nothing + push!(output, Model.ORDER_IEEE) + else + push!(output, Model.ORDER_FUTURE) + end + end + return output +end + +function _modelboundstate(state::Model.BoundState) + state == Model.BOUND_ABSENT && return :absent + state == Model.BOUND_UNKNOWN && return :unknown + state == Model.BOUND_KNOWN && return :known + throw(ArgumentError("unknown model bound state $state")) +end + +function _modelexactness(exactness::Model.Exactness) + exactness == Model.EXACTNESS_UNKNOWN && return :unknown + exactness == Model.EXACTNESS_INEXACT && return :inexact + exactness == Model.EXACTNESS_EXACT && return :exact + throw(ArgumentError("unknown model exactness $exactness")) +end + +function _modelfamily(family::Model.BoundFamily) + family == Model.FAMILY_NONE && return :none + family == Model.FAMILY_MODERN && return :modern + family == Model.FAMILY_DEPRECATED && return :deprecated + throw(ArgumentError("unknown model family $family")) +end + +function _modeltrust(state::Model.TrustState) + state == Model.TRUST_TRUSTED && return :trusted + state == Model.TRUST_UNTRUSTED && return :untrusted + throw(ArgumentError("unknown model trust state $state")) +end + +function _modelcomparison(comparison::Model.ComparatorKind) + comparison == Model.COMPARATOR_SIGNED && return :signed + comparison == Model.COMPARATOR_UNSIGNED && return :unsigned + comparison == Model.COMPARATOR_UNSIGNED_BYTES && return :unsigned_bytes + comparison == Model.COMPARATOR_DECIMAL && return :decimal + comparison == Model.COMPARATOR_BOOLEAN && return :boolean + comparison == Model.COMPARATOR_TYPE_FLOAT && return :floating + comparison == Model.COMPARATOR_IEEE_FLOAT && return :ieee_total_order + comparison == Model.COMPARATOR_UNDEFINED && return :undefined + throw(ArgumentError("unknown model comparator $comparison")) +end + +function _modeldeclared(order::Union{Nothing,Model.DeclaredOrder}) + order === nothing && return :absent + order == Model.ORDER_TYPE && return :type_order + order == Model.ORDER_IEEE && return :ieee_total_order + order == Model.ORDER_FUTURE && return :unknown + throw(ArgumentError("unknown model declared order $order")) +end + +function _modelordercomparison(spec::Model.LeafSpec, + order::Union{Nothing,Model.DeclaredOrder}) + order === nothing && return :undefined + order == Model.ORDER_FUTURE && return :undefined + order == Model.ORDER_IEEE && return :ieee_total_order + order == Model.ORDER_TYPE || throw(ArgumentError( + "unknown model declared order $order")) + return _modelcomparison(Model._typecomparator(spec)) +end + +function _modeloccupancy(occupancy::Model.OccupancyState) + occupancy == Model.OCCUPANCY_UNKNOWN && return :unknown + occupancy == Model.OCCUPANCY_EMPTY && return :no_non_null + occupancy == Model.OCCUPANCY_ALL_NAN && return :all_nan + occupancy == Model.OCCUPANCY_HAS_NON_NAN && return :has_non_nan + throw(ArgumentError("unknown model occupancy $occupancy")) +end + +function _productionboundreason(reason::Symbol) + reason === :valid && return :known + reason === :invalid_value && return :invalid_logical + reason in (:parquet_cpp_pre_1_3, :parquet_mr_pre_1_10) && + return :legacy_wrong_order + reason === :missing_order && return :missing_column_orders + reason === :unknown_order && return :unknown_column_order + reason === :undefined_order && return :undefined_type_order + reason === :no_non_null && return :no_non_null_values + reason === :ieee_bound_kind && return :ieee_bound_kind_contradiction + return reason +end + +function _productiontrustreason(reason::Symbol) + reason in (:parquet_cpp_pre_1_3, :parquet_mr_pre_1_10) && + return :legacy_wrong_order + reason === :affected_equal_bounds && return :trusted + return reason +end + +function _modeladjustment(reason::Symbol, name::String) + reason === :widened_zero || return :none + name == "lower" && return :negative_zero + name == "upper" && return :positive_zero + throw(ArgumentError("unknown statistic bound name $name")) +end + +function _comparecount(model::Model.CountFact, production, name::String) + expectedstate = model.known ? :known : :absent + _require(production.state == expectedstate, + "$name state differs from the independent model") + expected = model.known ? model.value : nothing + _require(production.value == expected, + "$name value differs from the independent model") + return +end + +function comparefacts(model::Model.StatisticsResult, production, + spec::Model.LeafSpec, order::Union{Nothing,Model.DeclaredOrder}) + _require(production.family == _modelfamily(model.family), + "statistics family differs from the independent model") + _require(production.order.declared == _modeldeclared(order), + "statistics declared order differs from the independent model") + _require(production.order.comparison == _modelordercomparison(spec, order), + "statistics declared-order comparison differs from the independent model") + expectedcomparison = _modelcomparison(model.comparator) + _require(production.comparison == expectedcomparison, + "statistics effective comparator $(production.comparison) differs from " * + "the independent model $expectedcomparison") + _require(production.occupancy == _modeloccupancy(model.occupancy), + "statistics occupancy differs from the independent model") + _require(production.trust.state == _modeltrust(model.trust.state), + "statistics trust differs from the independent model") + _require(_productiontrustreason(production.trust.reason) == model.trust.reason, + "statistics trust reason differs from the independent model") + for (name, modeled, actual) in (("lower", model.lower, production.lower), + ("upper", model.upper, production.upper)) + _require(actual.state == _modelboundstate(modeled.state), + "$name bound state differs from the independent model") + _require(actual.raw == modeled.raw, + "$name raw bound differs from the independent model") + _require(actual.exactness == _modelexactness(modeled.exactness), + "$name exactness differs from the independent model") + _require(_productionboundreason(actual.reason) == modeled.reason, + "$name reason differs from the independent model") + _require(actual.adjustment == _modeladjustment(modeled.reason, name), + "$name adjustment differs from the independent model") + end + _comparecount(model.null_count, production.null_count, "null_count") + _comparecount(model.nan_count, production.nan_count, "nan_count") + _comparecount(model.distinct_count, production.distinct_count, + "distinct_count") + return +end + +function _littlebytes(value::T) where {T<:Unsigned} + output = Vector{UInt8}(undef, sizeof(T)) + for index in eachindex(output) + output[index] = UInt8(value & T(0xff)) + value >>= 8 + end + return output +end + +function physicalraw(value, element::MD.SchemaElement) + physical = element.type_ + physical == MD.Type.BOOLEAN && return UInt8[value ? 0x01 : 0x00] + physical == MD.Type.INT32 && return _littlebytes( + reinterpret(UInt32, value::Int32)) + physical == MD.Type.INT64 && return _littlebytes( + reinterpret(UInt64, value::Int64)) + physical == MD.Type.FLOAT && return _littlebytes( + reinterpret(UInt32, value::Float32)) + physical == MD.Type.DOUBLE && return _littlebytes( + reinterpret(UInt64, value::Float64)) + physical == MD.Type.INT96 && return Vector{UInt8}(value) + physical in (MD.Type.BYTE_ARRAY, MD.Type.FIXED_LEN_BYTE_ARRAY) && + return Vector{UInt8}(value) + throw(ArgumentError("unsupported physical statistics value $physical")) +end + +function _samemodelvalue(left::Model.ModelValue, right::Model.ModelValue) + typeof(left) === typeof(right) || return false + left isa Model.SignedValue && return left.value == right.value + left isa Model.UnsignedValue && return left.value == right.value + left isa Model.BooleanValue && return left.value == right.value + left isa Model.ByteValue && return left.value == right.value + left isa Model.DecimalValue && return left.value == right.value + left isa Model.FloatValue && return left.width == right.width && + left.bits == right.bits + throw(ArgumentError("unsupported independent-model value $(typeof(left))")) +end + +function _summarycomparator(spec::Model.LeafSpec, + order::Union{Nothing,MD.ColumnOrder}) + order === nothing && return Model.COMPARATOR_UNDEFINED + order.IEEE_754_TOTAL_ORDER !== nothing && return Model.COMPARATOR_IEEE_FLOAT + order.TYPE_ORDER !== nothing && return Model._typecomparator(spec) + return Model.COMPARATOR_UNDEFINED +end + +function _checksummary(caseid::String, spec::Model.LeafSpec, + stream::Parquet.LeafStream, element::MD.SchemaElement, + order::Union{Nothing,MD.ColumnOrder}, result::Model.StatisticsResult, + statistics::Union{Nothing,MD.Statistics}, limit::Int64) + rawvalues = Vector{UInt8}[physicalraw(value, element) for value in stream.values] + comparator = _summarycomparator(spec, order) + comparator == Model.COMPARATOR_UNDEFINED && return + summary = Model.summarize_raw_values(spec, rawvalues, comparator) + if statistics !== nothing && statistics.nan_count !== nothing + _require(summary.nan_count == statistics.nan_count, + "nan_count differs from the independent raw-value summary") + end + startswith(caseid, "julia-reader-no-pruning-") && return + lowerraw = result.lower.raw + upperraw = result.upper.raw + if lowerraw === nothing || upperraw === nothing + _require(lowerraw === nothing && upperraw === nothing, + "writer emitted only one statistics bound") + if caseid == "julia-writer-oversized-bounds" + _require(summary.lower isa Model.ByteValue && + summary.upper isa Model.ByteValue, + "oversized variable summary has the wrong model type") + _require(!Model.writer_bounds_allowed(summary.lower.value, + summary.upper.value, Model.ModelLimits( + max_statistics_value_bytes=limit)), + "one-byte-over summary unexpectedly fits the statistics limit") + end + return + end + expectedlower = Model._summaryvalue(lowerraw, spec) + expectedupper = Model._summaryvalue(upperraw, spec) + _require(summary.lower !== nothing && summary.upper !== nothing, + "emitted bounds exist for an empty independent summary") + _require(_samemodelvalue(summary.lower, expectedlower), + "emitted lower bound is not the independent raw-value extremum") + _require(_samemodelvalue(summary.upper, expectedupper), + "emitted upper bound is not the independent raw-value extremum") + values = Model.ModelValue[Model._summaryvalue(raw, spec) for raw in rawvalues] + hasnonnull = any(value -> !(value isa Model.FloatValue) || + !Model.float_isnan(value), values) + for value in values + candidate = !(value isa Model.FloatValue) || + !Model.float_isnan(value) || + (comparator == Model.COMPARATOR_IEEE_FLOAT && !hasnonnull) + candidate || continue + _require(Model.contains_value(summary, value, comparator), + "independent extrema do not contain a contributing value") + end + return +end + +function _assertpageboundaries(file::Parquet.File, metadata::MD.FileMetaData) + for group in metadata.row_groups + for chunk in group.columns + _require(chunk.column_index_offset === nothing && + chunk.column_index_length === nothing, + "generated N6 fixture contains ColumnIndex content") + column = something(chunk.meta_data) + start, stop = Parquet._chunkrange(column, file.footer.offset) + position = start + while position < stop + frame = Parquet.readpage(file.source, position, stop, + Parquet.Limits()) + kind = Parquet.pagekind(frame) + if kind === :data_v1 + _require(frame.header.data_page_header.statistics === nothing, + "generated V1 page header contains statistics") + elseif kind === :data_v2 + _require(frame.header.data_page_header_v2.statistics === nothing, + "generated V2 page header contains statistics") + end + next = Parquet.pageend(frame) + _require(next > position, + "generated page walk did not advance") + position = next + end + _require(position == stop, + "generated page walk missed the column boundary") + end + end + return +end + +function _bodybytes(bytes::Vector{UInt8}) + _, offset, _ = _decodefooter(bytes) + return copy(@view bytes[1:offset]) +end + +function _materializedrows(bytes::Vector{UInt8}) + table = Parquet.Table(bytes) + try + return logicalrows(table.columns) + finally + close(table) + end +end + +function _tracedmaterialization(bytes::Vector{UInt8}) + source = TraceSource(bytes, Tuple{Int64,Int64}[]) + file = Parquet.File(source) + footerstart = file.footer.offset + try + TH.decode(copy(file.footer.bytes), MD.FileMetaData) + finally + close(file) + end + empty!(source.reads) + table = Parquet.Table(source) + rows = try + logicalrows(table.columns) + finally + close(table) + end + trace = IOBuffer() + count = 0 + for (offset, length_) in source.reads + readstop = Base.checked_add(offset, length_) + clippedstart = max(offset, Int64(4)) + clippedstop = min(readstop, footerstart) + clippedstop > clippedstart || continue + write(trace, "offset=", string(clippedstart), ";length=", + string(clippedstop - clippedstart), "\n") + count += 1 + end + tracehash = bytehash(take!(trace)) + bodyhash = bytehash(@view bytes[1:Int(footerstart)]) + logicalhash = canonicalhash(rows) + contract = "body_sha256=$bodyhash\n" * + "logical_values_sha256=$logicalhash\n" * + "range_trace_sha256=$tracehash\n" * + "read_count=$count\n" + return bytehash(codeunits(contract)), bodyhash, logicalhash, tracehash, count +end + +function _createdbyrecord(metadata::MD.FileMetaData) + return metadata.created_by !== nothing, metadata.created_by +end + +function _filerecord(declaration::Dict{String,Any}, bytes::Vector{UInt8}, + metadata::MD.FileMetaData, footerlength::Int64, leafcount::Int) + present, createdby = _createdbyrecord(metadata) + orders = metadata.column_orders + return Dict{String,Any}( + "record" => "file", + "schema_version" => 2, + "case_id" => declaration["id"], + "file" => declaration["output_file"], + "sha256" => bytehash(bytes), + "size" => length(bytes), + "footer_length" => Int(footerlength), + "row_group_count" => length(metadata.row_groups), + "leaf_count" => leafcount, + "column_order_count" => orders === nothing ? nothing : length(orders), + "created_by_present" => present, + "created_by" => createdby, + ) +end + +function _orderstate(order::Union{Nothing,MD.ColumnOrder}) + order === nothing && return :absent + order.TYPE_ORDER !== nothing && return :type + order.IEEE_754_TOTAL_ORDER !== nothing && return :ieee + return :future +end + +function _assertmodernstatistics(statistics::MD.Statistics) + _require(statistics.min === nothing && statistics.max === nothing, + "Julia writer emitted deprecated min/max statistics") + if statistics.min_value === nothing || statistics.max_value === nothing + _require(statistics.min_value === nothing && statistics.max_value === nothing, + "Julia writer emitted only one modern bound") + _require(statistics.is_min_value_exact === nothing && + statistics.is_max_value_exact === nothing, + "omitted modern bounds retain exactness flags") + else + _require(statistics.is_min_value_exact === true && + statistics.is_max_value_exact === true, + "Julia writer modern bounds are not marked exact") + end + return +end + +function _floatlogicalbits(value::Float16) + return UInt64(reinterpret(UInt16, value)) +end + +function _floatlogicalbits(value::Float32) + return UInt64(reinterpret(UInt32, value)) +end + +function _floatlogicalbits(value::Float64) + return reinterpret(UInt64, value) +end + +function _assertieeeprofile(profile::ProfileOutput, + interpretations::Vector{Any}) + for column in values(profile.table) + mixed = collect(skipmissing(column[1:6])) + allnan = collect(skipmissing(column[7:12])) + _require(count(ismissing, column[1:6]) == 1, + "IEEE mixed group does not contain one null") + _require(any(value -> iszero(value) && signbit(value), mixed) && + any(value -> iszero(value) && !signbit(value), mixed), + "IEEE mixed group does not contain both signed zeros") + _require(any(value -> isfinite(value) && !iszero(value), mixed), + "IEEE mixed group does not contain a finite nonzero value") + _require(any(isinf, mixed) && any(isnan, mixed), + "IEEE mixed group does not contain infinity and NaN") + _require(length(allnan) == 6 && all(isnan, allnan), + "IEEE all-NaN group contains a non-NaN value") + _require(length(unique(_floatlogicalbits.(allnan))) == 6, + "IEEE all-NaN group lacks distinct raw payloads") + end + for item in interpretations + expectednulls = item.group == 1 ? Int64(1) : Int64(0) + expectednans = item.group == 1 ? Int64(1) : Int64(6) + _require(item.statistics.null_count == expectednulls && + item.statistics.nan_count == expectednans, + "IEEE group count state differs from its profile") + end + return +end + +function _profileassertions(declaration::Dict{String,Any}, + profile::ProfileOutput, metadata::MD.FileMetaData, + interpretations::Vector{Any}) + caseid = declaration["id"] + orders = metadata.column_orders + states = orders === nothing ? Symbol[] : _orderstate.(orders) + statistics = Any[item.statistics for item in interpretations] + if startswith(caseid, "julia-writer-") && profile.statistics_enabled + _require(orders !== nothing && length(orders) == declaration["leaf_count"], + "statistics-enabled writer output lacks leaf-aligned column_orders") + for value in statistics + value === nothing || _assertmodernstatistics(value) + end + end + if caseid == "julia-writer-type-order" + _require(all(==( :type), states), + "type-order fixture has a non-TYPE_ORDER leaf") + elseif caseid == "julia-writer-ieee-order" + _require(all(==( :ieee), states), + "IEEE fixture has a non-IEEE leaf") + _require(all(item -> item.statistics.nan_count !== nothing, + interpretations), "IEEE fixture omits nan_count") + _assertieeeprofile(profile, interpretations) + elseif caseid == "julia-writer-undefined-order" + _require(all(==( :type), states), + "undefined-order fixture lacks TYPE_ORDER placeholders") + _require(all(item -> item.statistics !== nothing && + item.statistics.min_value === nothing && + item.statistics.max_value === nothing, interpretations), + "undefined-order fixture contains a bound") + elseif caseid == "julia-writer-statistics-disabled" + _require(orders === nothing, + "statistics=false emitted column_orders") + _require(all(isnothing, statistics), + "statistics=false emitted row-group statistics") + comparison = something(profile.comparison_bytes) + _require(_bodybytes(profile.bytes) == _bodybytes(comparison), + "statistics=false changed pre-footer bytes") + elseif caseid == "julia-writer-oversized-bounds" + _require(states == [:type], + "oversized fixture lacks TYPE_ORDER") + _require(all(item -> item.statistics !== nothing && + item.statistics.null_count !== nothing && + item.statistics.min_value === nothing && + item.statistics.max_value === nothing, interpretations), + "oversized fixture did not retain counts while omitting both bounds") + elseif caseid == "julia-writer-nested-row-groups" + _require(all(==( :type), states), + "nested fixture has a non-TYPE_ORDER leaf") + _require(any(item -> item.column.num_values != + metadata.row_groups[item.group].num_rows, interpretations), + "nested fixture does not prove leaf-entry counts") + elseif caseid == "julia-reader-untrusted-producer" + for item in interpretations + if item.leaf == 1 + _require(item.model.trust.state == Model.TRUST_UNTRUSTED, + "PARQUET-251 string bounds were not suppressed") + _require(item.model.lower.state == Model.BOUND_UNKNOWN && + item.model.upper.state == Model.BOUND_UNKNOWN, + "untrusted string bounds remained known") + else + _require(item.model.family == Model.FAMILY_DEPRECATED, + "legacy signed fixture did not select deprecated bounds") + _require(item.model.trust.state == Model.TRUST_TRUSTED, + "valid deprecated signed bounds became untrusted") + end + end + elseif caseid == "julia-reader-no-pruning-absent" + _require(orders === nothing && all(isnothing, statistics), + "absent no-pruning fixture retains statistics metadata") + elseif caseid == "julia-reader-no-pruning-trusted" + _require(all(item -> item.model.trust.state == Model.TRUST_TRUSTED && + item.model.lower.state == Model.BOUND_KNOWN && + item.model.upper.state == Model.BOUND_KNOWN, interpretations), + "trusted no-pruning fixture lacks known bounds") + elseif caseid == "julia-reader-no-pruning-untrusted" + _require(all(item -> item.model.trust.state == Model.TRUST_UNTRUSTED && + item.model.lower.state == Model.BOUND_UNKNOWN && + item.model.upper.state == Model.BOUND_UNKNOWN, interpretations), + "producer-untrusted no-pruning fixture retained known bounds") + elseif caseid == "julia-reader-no-pruning-oversized" + _require(all(item -> item.model.lower.state == Model.BOUND_UNKNOWN && + item.model.upper.state == Model.BOUND_UNKNOWN, interpretations), + "oversized no-pruning bounds remained known") + elseif caseid == "julia-reader-no-pruning-invalid" + _require(all(item -> item.model.lower.state == Model.BOUND_UNKNOWN, + interpretations), + "semantically invalid JSON lower bound remained known") + end + families = String[string(item.production.family) for item in interpretations] + trusts = String[string(item.production.trust.state) for item in interpretations] + return Dict{String,Any}( + "body_sha256" => bytehash(_bodybytes(profile.bytes)), + "column_orders" => String[string(state) for state in states], + "statistics_families" => families, + "trust_states" => trusts, + ) +end + +function checkcase(declaration::Dict{String,Any}, profile::ProfileOutput) + bytes = profile.bytes + length(bytes) <= MAX_GENERATED_BYTES || throw(ArgumentError( + "generated case exceeds the file-size limit")) + relative = declaration["output_file"] + caseid = declaration["id"] + expectedrows = logicalrows(profile.table) + expectedcanonical = canonicaljson(expectedrows) + file = Parquet.File(bytes) + metadata = nothing + schema = nothing + footerbytes = UInt8[] + footerlength = Int64(0) + records = Dict{String,Any}[] + interpretations = [] + try + footerbytes = copy(file.footer.bytes) + footerlength = file.footer.length + metadata = TH.decode(footerbytes, MD.FileMetaData) + _require(TH.encode(metadata) == footerbytes, + "decoded generated footer does not round-trip byte-for-byte") + schema = Parquet.Schema(metadata) + _require(metadata.num_rows == length(expectedrows), + "generated file row count differs from its profile") + _require(length(metadata.row_groups) == declaration["row_group_count"], + "generated row-group topology differs from fixtures.toml") + _require(length(schema.leaves) == declaration["leaf_count"], + "generated leaf topology differs from fixtures.toml") + _assertpageboundaries(file, metadata) + orders = metadata.column_orders + modeledorders = modelorders(orders) + for (groupindex, group) in enumerate(metadata.row_groups) + _require(length(group.columns) == length(schema.leaves), + "generated row group is not leaf aligned") + for leafindex in eachindex(schema.leaves) + node = schema.leaves[leafindex] + chunk = group.columns[leafindex] + column = something(chunk.meta_data) + order = orders === nothing ? nothing : orders[leafindex] + record = normalizecolumn(caseid, relative, groupindex, + leafindex, node, column, order) + push!(records, record) + spec = modelspec(record["leaf_schema"]) + modeled = Model.interpret_statistics(spec, column.num_values, + modelstatistics(column.statistics), modeledorders; + leaf_index=leafindex, leaf_count=length(schema.leaves), + created_by=metadata.created_by, + limits=Model.ModelLimits(max_statistics_value_bytes= + profile.statistics_limit)) + production = Parquet._statisticsfacts(schema, leafindex, + metadata.created_by, orders, column; + limits=Parquet.Limits(max_statistics_value_bytes= + profile.statistics_limit)) + modeledorder = modeledorders === nothing ? nothing : + modeledorders[leafindex] + comparefacts(modeled, production, spec, modeledorder) + stream = Parquet.readleafstream(file, metadata, schema, + groupindex, leafindex; expected_rows=group.num_rows) + _require(length(stream) == column.num_values, + "physical leaf entry count differs from num_values") + if column.statistics !== nothing + _require(column.statistics.null_count == + column.num_values - length(stream.values), + "null_count differs from entry count minus dense count") + end + _checksummary(caseid, spec, stream, node.element, order, + modeled, column.statistics, profile.statistics_limit) + push!(interpretations, (group=groupindex, leaf=leafindex, + node=node, column=column, statistics=column.statistics, + model=modeled, production=production, + dense_count=length(stream.values), + entry_count=length(stream))) + end + end + finally + close(file) + end + actualrows = _materializedrows(bytes) + actualcanonical = canonicaljson(actualrows) + _require(actualcanonical == expectedcanonical, + "generated fixture logical values changed after round trip") + assertions = _profileassertions(declaration, profile, metadata, + interpretations) + assertions["logical_values_sha256"] = bytehash(codeunits(actualcanonical)) + assertions["metadata_sha256"] = bytehash(footerbytes) + no_pruning = nothing + if !isempty(declaration["comparison_group"]) + no_pruning, bodyhash, logicalhash, tracehash, readcount = + _tracedmaterialization(bytes) + _require(logicalhash == assertions["logical_values_sha256"], + "traced and ordinary logical materialization differ") + assertions["body_sha256"] = bodyhash + assertions["range_trace_sha256"] = tracehash + assertions["read_count"] = readcount + end + file_record = _filerecord(declaration, bytes, metadata, footerlength, + length(schema.leaves)) + return CheckedCase(declaration, bytes, file_record, records, + assertions["logical_values_sha256"], assertions["metadata_sha256"], + assertions, no_pruning) +end + +function _parquetjlauthority(capabilities::Dict{String,Any}) + for authority in capabilities["authority"] + authority["id"] == "parquet-jl" && return authority + end + throw(ArgumentError("capabilities.toml lacks the parquet-jl authority")) +end + +function _reviewedclaimpairs(capabilities::Dict{String,Any}) + authority = _parquetjlauthority(capabilities) + pairs = Set{Tuple{String,String}}() + for claim in authority["claim"] + claim["status"] in ("planned", "verified") || continue + capability = claim["capability"] + for caseid in claim["cases"] + pair = (caseid, capability) + pair in pairs && throw(ArgumentError( + "duplicate parquet-jl claim $pair")) + push!(pairs, pair) + end + end + return authority, pairs +end + +function _standarddigest(case::CheckedCase, capability::String) + observations = Any[ + Dict{String,Any}("file_sha256" => case.file_record["sha256"]), + Dict{String,Any}("metadata_sha256" => case.metadata_sha256), + Dict{String,Any}("logical_values_sha256" => + case.logical_values_sha256), + Dict{String,Any}("assertions" => case.assertion_facts), + ] + envelope = Dict{String,Any}( + "capability_id" => capability, + "case_id" => case.declaration["id"], + "observations" => observations, + ) + return canonicalhash(envelope) +end + +function _caseresult(case::CheckedCase, capability::String) + contract = case.declaration["digest_contract"] + digest = if contract == "n6-no-pruning-trace-sha256-v1" + something(case.no_pruning_sha256) + elseif contract == "n6-capability-result-sha256-v1" + _standarddigest(case, capability) + else + throw(ArgumentError("unsupported digest contract $contract")) + end + return Dict{String,Any}( + "record" => "case_result", + "schema_version" => 2, + "case_id" => case.declaration["id"], + "capability_id" => capability, + "digest_contract" => contract, + "status" => "PASS", + "expected_sha256" => digest, + "actual_sha256" => digest, + "detail" => "Parquet.jl N6 harness assertion passed", + ) +end + +function _runrecord(authority::Dict{String,Any}, inputs::Dict{String,Vector{UInt8}}) + CANONICAL_TOOLCHAIN_SHA256 in authority["toolchain_sha256"] || + throw(ArgumentError("Parquet.jl producer descriptor is not pinned")) + authority["revision"] == CANONICAL_SOURCE_REVISION || + throw(ArgumentError("Parquet.jl source composite is not pinned")) + return Dict{String,Any}( + "record" => "run", + "schema_version" => 2, + "evidence_id" => "normalized-parquet-jl", + "producer" => authority["id"], + "producer_version" => authority["version"], + "source_revision" => authority["revision"], + "plan_sha256" => bytehash(inputs[PLAN_FILE]), + "capabilities_sha256" => bytehash(inputs[CAPABILITIES_FILE]), + "fixture_manifest_sha256" => bytehash(inputs[FIXTURES_FILE]), + "corpus_manifest_sha256" => bytehash(inputs[CORPUS_MANIFEST_FILE]), + "evidence_schema_sha256" => bytehash(inputs[EVIDENCE_SCHEMA_FILE]), + "toolchain_sha256" => CANONICAL_TOOLCHAIN_SHA256, + "unsupported_cases" => String[], + ) +end + +function _evidencebytes(records::Vector{Dict{String,Any}}) + output = IOBuffer() + for record in records + line = canonicaljson(record) + ncodeunits(line) <= 1024 * 1024 || throw(ArgumentError( + "normalized evidence line exceeds one MiB")) + write(output, line, '\n') + end + bytes = take!(output) + length(bytes) <= 32 * 1024 * 1024 || throw(ArgumentError( + "normalized evidence exceeds 32 MiB")) + return bytes +end + +function _assertcomparisongroup(checked::Vector{CheckedCase}) + grouped = CheckedCase[case for case in checked if + case.declaration["comparison_group"] == "julia-reader-no-pruning-v1"] + length(grouped) == 5 || throw(AssertionError( + "no-pruning comparison group does not have five variants")) + digests = Set(something(case.no_pruning_sha256) for case in grouped) + length(digests) == 1 || throw(AssertionError( + "no-pruning variants have different exact digest contracts")) + fields = ("body_sha256", "logical_values_sha256", + "range_trace_sha256", "read_count") + for field in fields + values = Set(case.assertion_facts[field] for case in grouped) + length(values) == 1 || throw(AssertionError( + "no-pruning variants differ in $field")) + end + return +end + +function _saferepositoryfile(relative::String) + isabspath(relative) && throw(ArgumentError( + "frozen model path is absolute: $relative")) + occursin('\\', relative) && throw(ArgumentError( + "frozen model path contains a backslash: $relative")) + parts = split(relative, '/') + any(part -> isempty(part) || part in (".", ".."), parts) && + throw(ArgumentError("frozen model path is unsafe: $relative")) + path = normpath(joinpath(REPO_ROOT, parts...)) + startswith(relpath(path, REPO_ROOT), "..") && throw(ArgumentError( + "frozen model path escapes the repository: $relative")) + current = REPO_ROOT + for part in parts + current = joinpath(current, part) + islink(current) && throw(ArgumentError( + "frozen model path contains a symlink: $relative")) + end + return path +end + +function _checkmodelpins(manifest::Dict{String,Any}) + entries = manifest["frozen_model"] + isempty(entries) && throw(ArgumentError("manifest has no frozen N6 model")) + seen = Set{String}() + for entry in entries + relative = entry["file"] + relative in seen && throw(ArgumentError( + "duplicate frozen model path: $relative")) + push!(seen, relative) + path = _saferepositoryfile(relative) + bytes = _stablefilebytes(path, MAX_DECLARATION_BYTES, + "frozen model input $relative") + bytehash(bytes) == entry["sha256"] || throw(ArgumentError( + "frozen model input digest differs: $relative")) + path == MODEL_FILE && bytes != FROZEN_MODEL_BYTES && throw(ArgumentError( + "included N6 model bytes differ from the checked snapshot")) + end + modelrelative = replace(relpath(MODEL_FILE, REPO_ROOT), + Base.Filesystem.path_separator => '/') + modelrelative in seen || throw(ArgumentError( + "manifest does not pin the included N6 model source")) + only(entry for entry in entries if entry["file"] == modelrelative)["sha256"] == + FROZEN_MODEL_SHA256 || throw(ArgumentError( + "manifest model source hash differs from the bootstrap pin")) + return +end + +function _checkedinputs() + paths = (PLAN_FILE, CAPABILITIES_FILE, FIXTURES_FILE, MANIFEST_FILE, + EVIDENCE_SCHEMA_FILE, CORPUS_MANIFEST_FILE) + inputs = Dict{String,Vector{UInt8}}() + for path in paths + inputs[path] = _stablefilebytes(path, MAX_DECLARATION_BYTES, + "required N6 input $path") + end + manifest = TOML.parse(String(copy(inputs[MANIFEST_FILE]))) + fixtures = TOML.parse(String(copy(inputs[FIXTURES_FILE]))) + capabilities = TOML.parse(String(copy(inputs[CAPABILITIES_FILE]))) + _checkmodelpins(manifest) + return inputs, manifest, fixtures, capabilities +end + +function buildharness() + inputs, _, fixtures, capabilities = _checkedinputs() + _, declarations = checkeddeclarations(fixtures) + authority, pairs = _reviewedclaimpairs(capabilities) + checked = CheckedCase[] + files = Pair{String,Vector{UInt8}}[] + for declaration in declarations + profile = generateprofile(declaration) + case = checkcase(declaration, profile) + if declaration["output_identity_status"] == "verified" + _require(case.file_record["sha256"] == declaration["output_sha256"] && + case.file_record["size"] == declaration["output_size"], + "generated identity differs from the verified declaration for " * + declaration["id"]) + end + push!(checked, case) + push!(files, declaration["output_file"] => case.bytes) + end + _assertcomparisongroup(checked) + records = Dict{String,Any}[_runrecord(authority, inputs)] + results = 0 + for case in checked + push!(records, case.file_record) + append!(records, case.column_records) + declared = Set(String.(case.declaration["capabilities"])) + for capability in case.declaration["capabilities"] + pair = (case.declaration["id"], capability) + pair in pairs || continue + capability in declared || throw(AssertionError( + "reviewed capability is outside the generated declaration")) + push!(records, _caseresult(case, capability)) + results += 1 + end + end + length(checked) == 12 || throw(AssertionError( + "N6 harness did not build 12 generated cases")) + sum(length(case.column_records) for case in checked) == 52 || + throw(AssertionError("N6 harness did not build 52 column records")) + results == 31 || throw(AssertionError( + "N6 harness did not build 31 reviewed case results")) + length(records) == 96 || throw(AssertionError( + "N6 harness did not build 96 normalized evidence records")) + return HarnessOutput(files, _evidencebytes(records), checked) +end + +function _safeoutput(relative::String) + occursin(r"^generated/[A-Za-z0-9._+@=-]+$", relative) || + throw(ArgumentError("unsafe generated output path $relative")) + target = normpath(joinpath(N6_ROOT, split(relative, '/')...)) + dirname(target) == joinpath(N6_ROOT, "generated") || + throw(ArgumentError("generated output escapes its declared directory")) + return target +end + +function _safedirectory(path::String, create::Bool) + parent = dirname(path) + parent in (joinpath(N6_ROOT, "generated"), + joinpath(N6_ROOT, "evidence")) || throw(ArgumentError( + "output directory is outside the N6 harness boundary")) + islink(parent) && throw(ArgumentError("output directory is a symlink")) + if ispath(parent) + isdir(parent) || throw(ArgumentError("output parent is not a directory")) + elseif create + mkdir(parent) + else + throw(ArgumentError("output directory is absent: $parent")) + end + islink(parent) && throw(ArgumentError("output directory became a symlink")) + return +end + +function _preflighttarget(path::String) + isdir(dirname(path)) || throw(ArgumentError( + "atomic output parent is not a directory")) + islink(dirname(path)) && throw(ArgumentError( + "atomic output parent is a symlink")) + islink(path) && throw(ArgumentError("refusing to replace symlink output $path")) + ispath(path) && !isfile(path) && throw(ArgumentError( + "refusing to replace non-file output $path")) + return +end + +function _fsyncdescriptor(descriptor, label::String) + Sys.isunix() || throw(ArgumentError( + "durable N6 output publication requires a POSIX host")) + errorcode = ccall(:fsync, Cint, (Cint,), descriptor) + iszero(errorcode) || throw(SystemError("fsync $label", Libc.errno())) + return +end + +function _fsyncdirectory(path::String) + Sys.isunix() || throw(ArgumentError( + "durable N6 output publication requires a POSIX host")) + descriptor = ccall(:open, Cint, (Cstring, Cint), path, 0) + descriptor >= 0 || throw(SystemError( + "open output directory $(repr(path))", Libc.errno())) + try + _fsyncdescriptor(descriptor, "output directory $(repr(path))") + finally + ccall(:close, Cint, (Cint,), descriptor) + end + return +end + +function _renamefile(source::String, target::String) + errorcode = ccall(:jl_fs_rename, Int32, (Cstring, Cstring), source, target) + iszero(errorcode) || throw(SystemError( + "atomic rename $(repr(source)) to $(repr(target))", errorcode)) + return +end + +function _stagebytes(path::String, bytes::Vector{UInt8}) + temporary, stream = mktemp(dirname(path); cleanup=false) + staged = false + try + write(stream, bytes) + flush(stream) + chmod(temporary, 0o644) + _fsyncdescriptor(fd(stream), "staged output $(repr(path))") + close(stream) + staged = true + return temporary + finally + isopen(stream) && close(stream) + !staged && ispath(temporary) && rm(temporary; force=true) + end +end + +function _reservedtemppath(parent::String) + path, stream = mktemp(parent; cleanup=false) + close(stream) + return path +end + +function _renameoverreserved(source::String, parent::String; + renamefile=_renamefile) + reserved = _reservedtemppath(parent) + renamed = false + try + renamefile(source, reserved) + renamed = true + return reserved + finally + !renamed && ispath(reserved) && rm(reserved; force=true) + end +end + +function _rollbackbatch!(installed::Vector{String}, + backups::Dict{String,Union{Nothing,String}}, syncdirectory) + for path in Iterators.reverse(installed) + replacement = _renameoverreserved(path, dirname(path)) + backup = backups[path] + backup === nothing || _renamefile(backup, path) + backups[path] = nothing + syncdirectory(dirname(path)) + rm(replacement; force=true) + syncdirectory(dirname(path)) + end + installedset = Set(installed) + for (path, backup) in backups + path in installedset && continue + backup === nothing && continue + ispath(path) && throw(ErrorException( + "cannot restore atomic output backup over an existing path")) + _renamefile(backup, path) + backups[path] = nothing + syncdirectory(dirname(path)) + end + return +end + +function _stagetargets(targets::Vector{Pair{String,Vector{UInt8}}}) + stages = Dict{String,String}() + try + for (path, bytes) in targets + stages[path] = _stagebytes(path, bytes) + end + catch + for stage in values(stages) + ispath(stage) && rm(stage; force=true) + end + rethrow() + end + return stages +end + +function _committarget!(path::String, stages::Dict{String,String}, + backups::Dict{String,Union{Nothing,String}}, installed::Vector{String}, + syncdirectory) + backup = nothing + if isfile(path) + backup = _renameoverreserved(path, dirname(path)) + backups[path] = backup + syncdirectory(dirname(path)) + end + haskey(backups, path) || (backups[path] = nothing) + try + _renamefile(stages[path], path) + catch + if backup !== nothing && !ispath(path) + _renamefile(backup, path) + backups[path] = nothing + syncdirectory(dirname(path)) + end + rethrow() + end + delete!(stages, path) + push!(installed, path) + syncdirectory(dirname(path)) + return +end + +function _cleanupbackups!(backups::Dict{String,Union{Nothing,String}}, + syncdirectory) + for backup in values(backups) + backup === nothing && continue + try + ispath(backup) && rm(backup; force=true) + syncdirectory(dirname(backup)) + catch error + @warn "could not remove committed output backup" backup exception=error + end + end + return +end + +function _atomicreplacebatch(targets::Vector{Pair{String,Vector{UInt8}}}; + syncdirectory=_fsyncdirectory) + paths = String[first(target) for target in targets] + length(unique(paths)) == length(paths) || throw(ArgumentError( + "atomic output targets are not unique")) + foreach(_preflighttarget, paths) + stages = _stagetargets(targets) + backups = Dict{String,Union{Nothing,String}}() + installed = String[] + try + for path in paths + _committarget!(path, stages, backups, installed, syncdirectory) + end + catch error + try + _rollbackbatch!(installed, backups, syncdirectory) + catch rollback + throw(ErrorException("atomic output rollback failed after " * + "$(sprint(showerror, error)): $(sprint(showerror, rollback))")) + finally + for stage in values(stages) + ispath(stage) && rm(stage; force=true) + end + end + rethrow() + end + _cleanupbackups!(backups, syncdirectory) + return +end + +function _atomicreplacebytes(path::String, bytes::Vector{UInt8}) + _atomicreplacebatch(Pair{String,Vector{UInt8}}[path => bytes]) + return +end + +function _atomicreplace(path::String, bytes::Vector{UInt8}) + _safedirectory(path, true) + _atomicreplacebytes(path, bytes) + return +end + +function _checkbytes(path::String, expected::Vector{UInt8}) + _safedirectory(path, false) + actual = _stablefilebytes(path, Int64(length(expected)), + "checked output $path") + actual == expected || throw(AssertionError("stale generated output: $path")) + return +end + +function _producersourcepaths() + project = joinpath(REPO_ROOT, "Project.toml") + isfile(project) && !islink(project) || throw(ArgumentError( + "Parquet.jl Project.toml is not a regular file")) + root = joinpath(REPO_ROOT, "src") + isdir(root) && !islink(root) || throw(ArgumentError( + "Parquet.jl source root is not a directory")) + paths = String["Project.toml"] + for (directory, directories, files) in walkdir(root; follow_symlinks=false) + for name in directories + path = joinpath(directory, name) + islink(path) && throw(ArgumentError( + "Parquet.jl source directory is a symbolic link")) + end + for name in files + path = joinpath(directory, name) + relative = replace(relpath(path, REPO_ROOT), + Base.Filesystem.path_separator => '/') + isfile(path) && !islink(path) || throw(ArgumentError( + "Parquet.jl source entry is not a regular file: $relative")) + push!(paths, relative) + end + end + sort!(paths) + return paths +end + +function _producerfilemap() + files = Dict{String,String}() + for item in PRODUCER_DESCRIPTOR["file"] + path = item["path"] + haskey(files, path) && throw(ArgumentError( + "duplicate Parquet.jl producer file: $path")) + files[path] = item["sha256"] + end + return files +end + +function _producersourcecomposite(paths::Vector{String}, payloads) + context = SHA.SHA2_256_CTX() + SHA.update!(context, + codeunits(PRODUCER_DESCRIPTOR["source_composite_algorithm"] * "\0")) + total = Int64(0) + for path in paths + bytes = payloads[path] + total = Base.checked_add(total, Int64(length(bytes))) + SHA.update!(context, codeunits("F\0")) + SHA.update!(context, codeunits(path)) + SHA.update!(context, codeunits("\0")) + SHA.update!(context, codeunits(string(length(bytes)))) + SHA.update!(context, codeunits("\0")) + SHA.update!(context, SHA.sha256(bytes)) + end + return bytes2hex(SHA.digest!(context)), total +end + +function _checkproduceridentity() + current = _stablefilebytes(PRODUCER_DESCRIPTOR_FILE, + Int64(4 * 1024 * 1024), "Parquet.jl producer descriptor") + current == PRODUCER_DESCRIPTOR_BYTES || throw(ArgumentError( + "Parquet.jl producer descriptor changed after package loading")) + files = _producerfilemap() + sourcepaths = _producersourcepaths() + support = Set(String[PRODUCER_DESCRIPTOR[field] for field in + ("manifest_file", "harness_file", "runner_file", "bootstrap_file")]) + Set(keys(files)) == union(Set(sourcepaths), support) || throw(ArgumentError( + "Parquet.jl producer file inventory differs")) + payloads = Dict{String,Vector{UInt8}}() + for (relative, digest) in files + path = joinpath(REPO_ROOT, split(relative, '/')...) + bytes = _stablefilebytes(path, Int64(64 * 1024 * 1024), + "Parquet.jl producer file $relative") + bytehash(bytes) == digest || throw(ArgumentError( + "Parquet.jl producer file digest differs: $relative")) + payloads[relative] = bytes + end + composite, total = _producersourcecomposite(sourcepaths, payloads) + composite == CANONICAL_SOURCE_REVISION || throw(ArgumentError( + "Parquet.jl source composite differs")) + length(sourcepaths) == PRODUCER_DESCRIPTOR["source_file_count"] || + throw(ArgumentError("Parquet.jl source file count differs")) + total == PRODUCER_DESCRIPTOR["source_total_bytes"] || + throw(ArgumentError("Parquet.jl source byte count differs")) + return +end + +function _checkwriteridentity() + VERSION == CANONICAL_WRITER_VERSION || throw(ArgumentError( + "N6 evidence writes require Julia $CANONICAL_WRITER_VERSION")) + _checkproduceridentity() + executable = String(Base.julia_cmd().exec[1]) + digest = filehash(executable; maximum=MAX_DECLARATION_BYTES, + label="Julia writer executable") + digest == CANONICAL_WRITER_EXECUTABLE_SHA256 || throw(ArgumentError( + "Julia writer executable digest differs from the frozen toolchain")) + return +end + +function runharness(mode::Symbol) + mode in (:write, :check) || throw(ArgumentError( + "N6 harness mode must be :write or :check")) + mode === :write && _checkwriteridentity() + output = buildharness() + targets = Pair{String,Vector{UInt8}}[ + _safeoutput(relative) => bytes for (relative, bytes) in output.files] + push!(targets, EVIDENCE_FILE => output.evidence) + if mode === :write + for (path, _) in targets + _safedirectory(path, true) + end + _atomicreplacebatch(targets) + else + for (path, bytes) in targets + _checkbytes(path, bytes) + end + end + return output +end + +end diff --git a/test/conformance/n6/julia/bootstrap.jl b/test/conformance/n6/julia/bootstrap.jl new file mode 100644 index 0000000..c1a082b --- /dev/null +++ b/test/conformance/n6/julia/bootstrap.jl @@ -0,0 +1,978 @@ +module N6ParquetJLBootstrap + +using SHA +using TOML + +const DESCRIPTOR_RELATIVE = "test/conformance/n6/julia/parquet-jl-producer.toml" +const DESCRIPTOR_SHA256_ENV = "PARQUET_N6_PRODUCER_DESCRIPTOR_SHA256" +const SOURCE_COMPOSITE_ALGORITHM = "parquet-jl-n6-source-composite-v1" +const SOURCE_COMPOSITE_PREFIX = "parquet-jl-n6-source-composite-v1\0" +const PACKAGE_NAME = "Parquet" +const PACKAGE_UUID = "626c502c-15b0-58ad-a749-f091afb673ae" +const CANONICAL_JULIA_VERSION = v"1.12.6" +const CANONICAL_PLATFORM = "macos-15-arm64" +const MAX_DESCRIPTOR_BYTES = Int64(4 * 1024 * 1024) +const MAX_PINNED_FILE_BYTES = Int64(64 * 1024 * 1024) +const MAX_SOURCE_FILES = 10_000 +const MAX_SOURCE_BYTES = Int64(512 * 1024 * 1024) +const MAX_TREE_ENTRIES = 100_000 +const MAX_TREE_FILE_BYTES = Int64(1024 * 1024 * 1024) +const MAX_TREE_BYTES = Int64(4) * 1024 * 1024 * 1024 +const MAX_RUNTIME_ENTRIES = 10_000 +const MAX_RUNTIME_FILE_BYTES = Int64(1024 * 1024 * 1024) +const MAX_RUNTIME_BYTES = Int64(2) * 1024 * 1024 * 1024 +const REQUIRED_ENVIRONMENT = Dict( + "HOME" => "/var/empty", + "JULIA_LOAD_PATH" => "@:@stdlib", + "JULIA_PKG_OFFLINE" => "true", + "JULIA_PKG_SERVER" => "", + "JULIA_PKG_PRECOMPILE_AUTO" => "0", +) +const ALLOWED_JULIA_ENVIRONMENT = Set([ + "JULIA_DEPOT_PATH", + "JULIA_LOAD_PATH", + "JULIA_NUM_THREADS", + "JULIA_PKG_OFFLINE", + "JULIA_PKG_PRECOMPILE_AUTO", + "JULIA_PKG_SERVER", +]) +const FORBIDDEN_LINKER_ENVIRONMENT = ( + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "LD_LIBRARY_PATH", + "LD_PRELOAD", +) +const PREFERENCE_FILES = ( + "JuliaLocalPreferences.toml", + "JuliaPreferences.toml", + "LocalPreferences.toml", + "Preferences.toml", +) + +struct TreeEntry + relative::String + path::String + metadata::Base.Filesystem.StatStruct + link_target::Union{Nothing,String} +end + +struct BootstrapContext + root::String + depot::String + runtime_root::String + descriptor_path::String + descriptor_sha256::String + descriptor::Dict{String,Any} + manifest::Dict{String,Any} + pinned_bytes::Dict{String,Vector{UInt8}} + dependency_roots::Dict{String,String} + artifact_roots::Dict{String,String} +end + +function _require(condition::Bool, message::AbstractString) + condition || throw(ArgumentError(message)) + return +end + +function _statidentity(value) + return (value.device, value.inode, value.mode, value.nlink, value.size, + value.mtime, value.ctime) +end + +function _filekind(metadata) + return metadata.mode & Base.Filesystem.S_IFMT +end + +function _regular(metadata) + return _filekind(metadata) == Base.Filesystem.S_IFREG +end + +function _directory(metadata) + return _filekind(metadata) == Base.Filesystem.S_IFDIR +end + +function _symlink(metadata) + return _filekind(metadata) == Base.Filesystem.S_IFLNK +end + +function _readonly(metadata) + return metadata.mode & 0o222 == 0 +end + +function _within(path::AbstractString, root::AbstractString) + path == root && return true + return startswith(path, root * Base.Filesystem.path_separator) +end + +function _portable(path::AbstractString) + return replace(path, Base.Filesystem.path_separator => '/') +end + +function _safe_relative(relative::String, label::String) + _require(!isempty(relative), "$label path is empty") + _require(!isabspath(relative), "$label path is absolute: $relative") + _require(!occursin('\\', relative), "$label path contains a backslash: $relative") + _require(all(byte -> byte <= 0x7f, codeunits(relative)), + "$label path is not ASCII: $relative") + _require(!occursin('\0', relative) && !occursin('\n', relative), + "$label path contains a control separator: $relative") + parts = split(relative, '/') + _require(all(part -> !isempty(part) && part != "." && part != "..", parts), + "$label path is unsafe: $relative") + return parts +end + +function _safe_path(root::String, relative::String, label::String) + parts = _safe_relative(relative, label) + current = root + for part in parts + current = joinpath(current, part) + _require(!islink(current), "$label path contains a symbolic link: $relative") + end + absolute = normpath(joinpath(root, parts...)) + _require(_within(absolute, root), "$label path escapes its root: $relative") + return absolute +end + +function _stable_file_bytes(path::String, maximum::Int64, label::String) + _require(maximum >= 0, "$label byte limit is negative") + _require(maximum < typemax(Int), "$label byte limit is too large") + before = lstat(path) + _require(_regular(before) && !islink(path), "$label is not a regular file") + _require(0 <= before.size <= maximum, "$label exceeds its byte limit") + return open(path, "r") do stream + opened = stat(stream) + _require(_statidentity(opened) == _statidentity(before), + "$label changed while it was opened") + bytes = read(stream, Int(maximum) + 1) + _require(length(bytes) <= maximum, "$label exceeds its byte limit") + _require(length(bytes) == opened.size && eof(stream), + "$label changed size while it was read") + final = stat(stream) + current = lstat(path) + _require(_regular(current) && !islink(path), + "$label changed type while it was read") + _require(_statidentity(final) == _statidentity(opened) && + _statidentity(current) == _statidentity(opened), + "$label changed while it was read") + return bytes + end +end + +function _stable_file_digest(path::String, maximum::Int64, label::String) + before = lstat(path) + _require(_regular(before) && !islink(path), "$label is not a regular file") + _require(0 <= before.size <= maximum, "$label exceeds its byte limit") + digest = open(path, "r") do stream + opened = stat(stream) + _require(_statidentity(opened) == _statidentity(before), + "$label changed while it was opened") + value = SHA.sha256(stream) + _require(eof(stream), "$label was not read to its end") + final = stat(stream) + current = lstat(path) + _require(_regular(current) && !islink(path), + "$label changed type while it was read") + _require(_statidentity(final) == _statidentity(opened) && + _statidentity(current) == _statidentity(opened), + "$label changed while it was read") + return value + end + return digest, Int64(before.size) +end + +function _hex_sha256(value, label::String) + _require(value isa String && occursin(r"^[0-9a-f]{64}$", value), + "$label is not a lowercase SHA-256 digest") + return value::String +end + +function _hex_sha1(value, label::String) + _require(value isa String && occursin(r"^[0-9a-f]{40}$", value), + "$label is not a lowercase Git tree SHA-1 digest") + return value::String +end + +function _string_field(table::AbstractDict, key::String, label::String) + _require(haskey(table, key), "$label lacks $key") + value = table[key] + _require(value isa String, "$label $key is not a string") + return value::String +end + +function _integer_field(table::AbstractDict, key::String, label::String) + _require(haskey(table, key), "$label lacks $key") + value = table[key] + _require(value isa Integer && !(value isa Bool), + "$label $key is not an integer") + return Int(value) +end + +function _table_array(table::AbstractDict, key::String, label::String) + _require(haskey(table, key), "$label lacks $key") + value = table[key] + _require(value isa AbstractVector, "$label $key is not an array") + _require(all(item -> item isa AbstractDict, value), + "$label $key contains a non-table value") + return value +end + +function _exact_keys(table::AbstractDict, expected, label::String) + actual = Set(String(key) for key in keys(table)) + required = Set(String(key) for key in expected) + _require(actual == required, "$label keys differ; expected $(sort!(collect(required))), " * + "got $(sort!(collect(actual)))") + return +end + +function _canonical_root(path::String, label::String) + absolute = normpath(abspath(path)) + _require(isdir(absolute) && !islink(absolute), "$label is not a directory") + _require(realpath(absolute) == absolute, "$label is not canonical") + return absolute +end + +function _validate_bootstrap_stdlibs() + _require(VERSION == CANONICAL_JULIA_VERSION, + "N6 bootstrap has the wrong Julia version") + runtime_root = _canonical_root(dirname(Sys.BINDIR), "Julia runtime root") + stdlib_root = joinpath(runtime_root, "share", "julia", "stdlib", "v1.12") + expected = ( + SHA => joinpath(stdlib_root, "SHA", "src", "SHA.jl"), + TOML => joinpath(stdlib_root, "TOML", "src", "TOML.jl"), + ) + for (module_, path) in expected + _require(pathof(module_) == path, + "bootstrap stdlib resolves outside the exact Julia runtime: $(nameof(module_))") + end + return +end + +function _validate_environment(root::String, depot::String) + for (key, expected) in REQUIRED_ENVIRONMENT + _require(get(ENV, key, nothing) == expected, + "$key differs from the isolated N6 value") + end + _require(get(ENV, "JULIA_DEPOT_PATH", nothing) == depot, + "JULIA_DEPOT_PATH is not the canonical private depot") + threads = get(ENV, "JULIA_NUM_THREADS", nothing) + _require(threads === nothing || threads == "1", + "JULIA_NUM_THREADS is not one") + for key in keys(ENV) + startswith(key, "JULIA_") || continue + _require(key in ALLOWED_JULIA_ENVIRONMENT, + "unexpected Julia environment input: $key") + end + for key in FORBIDDEN_LINKER_ENVIRONMENT + _require(!haskey(ENV, key), "forbidden dynamic-linker environment input: $key") + end + _require(Threads.nthreads() == 1, "N6 bootstrap requires one Julia thread") + _require(Threads.nthreads(:default) == 1 && Threads.nthreads(:interactive) == 0, + "N6 bootstrap requires one default thread and no interactive threads") + _require(Base.LOAD_PATH == ["@", "@stdlib"], + "Julia load path is not isolated") + _require(Base.DEPOT_PATH == [depot], "Julia depot path is not isolated") + _require(Base.active_project() == joinpath(root, "Project.toml"), + "active Julia project is not the read-only gate root") + options = Base.JLOptions() + _require(options.startupfile == 2, "Julia startup files are enabled") + _require(options.historyfile == 0, "Julia history is enabled") + _require(options.use_compiled_modules == 0, "Julia compiled modules are enabled") + _require(options.use_pkgimages == 0, "Julia package images are enabled") + _require(isempty(ARGS), "N6 bootstrap does not accept arguments") + return +end + +function _walk_entries(root::String, maximum_entries::Int, maximum_file_bytes::Int64, + maximum_total_bytes::Int64, label::String; safe_links::Bool) + entries = TreeEntry[] + total = Int64(0) + for (directory, directories, files) in walkdir(root; follow_symlinks=false) + sort!(directories) + sort!(files) + for name in vcat(directories, files) + path = joinpath(directory, name) + metadata = lstat(path) + if _symlink(metadata) + target = readlink(path) + _require(!occursin('\0', target) && !occursin('\n', target), + "$label symbolic-link target is invalid") + if safe_links + _require(!isabspath(target), + "$label has an absolute symbolic link: $path") + normalized = normpath(joinpath(dirname(path), target)) + _require(_within(normalized, root), + "$label symbolic link escapes its root: $path") + _require(ispath(path), "$label has a broken symbolic link: $path") + _require(_within(realpath(path), root), + "$label symbolic link resolves outside its root: $path") + end + relative = _portable(relpath(path, root)) + push!(entries, TreeEntry(relative, path, metadata, target)) + elseif _regular(metadata) + _require(0 <= metadata.size <= maximum_file_bytes, + "$label file exceeds its byte limit: $path") + _require(metadata.size <= maximum_total_bytes - total, + "$label exceeds its total byte limit") + total += metadata.size + relative = _portable(relpath(path, root)) + push!(entries, TreeEntry(relative, path, metadata, nothing)) + else + _require(_directory(metadata), "$label contains a special file: $path") + relative = _portable(relpath(path, root)) + push!(entries, TreeEntry(relative, path, metadata, nothing)) + end + _require(length(entries) <= maximum_entries, + "$label exceeds its entry limit") + end + end + sort!(entries; by=entry -> entry.relative) + _require(length(unique(entry.relative for entry in entries)) == length(entries), + "$label has duplicate paths") + return entries, total +end + +function _tree_identity(root::String, maximum_entries::Int, + maximum_file_bytes::Int64, maximum_total_bytes::Int64, label::String; + safe_links::Bool=true) + entries, total = _walk_entries(root, maximum_entries, maximum_file_bytes, + maximum_total_bytes, label; safe_links=safe_links) + context = SHA.SHA2_256_CTX() + for entry in entries + if _directory(entry.metadata) + continue + elseif entry.link_target === nothing + digest, _ = _stable_file_digest(entry.path, maximum_file_bytes, + "$label file $(entry.relative)") + SHA.update!(context, codeunits("F\0")) + SHA.update!(context, codeunits(entry.relative)) + SHA.update!(context, codeunits("\0")) + SHA.update!(context, codeunits(bytes2hex(digest))) + SHA.update!(context, codeunits("\n")) + else + current = lstat(entry.path) + _require(_symlink(current) && + _statidentity(current) == _statidentity(entry.metadata) && + readlink(entry.path) == entry.link_target, + "$label symbolic link changed while it was hashed") + SHA.update!(context, codeunits("L\0")) + SHA.update!(context, codeunits(entry.relative)) + SHA.update!(context, codeunits("\0")) + SHA.update!(context, codeunits(entry.link_target)) + SHA.update!(context, codeunits("\n")) + end + end + return (sha256=bytes2hex(SHA.digest!(context)), + entry_count=length(entries), total_bytes=total) +end + +function _tree_sha256(root::String, maximum_entries::Int, + maximum_file_bytes::Int64, maximum_total_bytes::Int64, label::String; + safe_links::Bool=true) + identity = _tree_identity(root, maximum_entries, maximum_file_bytes, + maximum_total_bytes, label; safe_links=safe_links) + return identity.sha256 +end + +function _assert_readonly_gate(root::String) + root_metadata = lstat(root) + _require(_directory(root_metadata) && _readonly(root_metadata), + "gate root is not read-only") + entries, _ = _walk_entries(root, MAX_TREE_ENTRIES, MAX_TREE_FILE_BYTES, + MAX_TREE_BYTES, "gate root"; safe_links=true) + for entry in entries + _require(entry.link_target === nothing, + "gate root contains a symbolic link: $(entry.relative)") + _require(_readonly(entry.metadata), + "gate root entry is writable: $(entry.relative)") + end + for relative in PREFERENCE_FILES + _require(!ispath(joinpath(root, relative)), + "gate root contains a preference input: $relative") + end + return +end + +function _assert_readonly_depot(depot::String) + root_metadata = lstat(depot) + _require(_directory(root_metadata) && _readonly(root_metadata), + "private depot root is not read-only") + entries, _ = _walk_entries(depot, MAX_TREE_ENTRIES, MAX_TREE_FILE_BYTES, + MAX_TREE_BYTES, "private depot"; safe_links=true) + for entry in entries + entry.link_target === nothing || continue + _require(_readonly(entry.metadata), + "private depot entry is writable: $(entry.relative)") + end + return +end + +function _validate_descriptor(descriptor::Dict{String,Any}) + expected = [ + "artifact", "bootstrap_file", "bootstrap_sha256", "dependency", + "descriptor_version", "file", "harness_file", "harness_sha256", + "julia_executable_sha256", "julia_runtime_entry_count", + "julia_runtime_total_bytes", "julia_runtime_tree_sha256", "julia_version", + "manifest_file", "manifest_sha256", "package_name", "package_uuid", + "package_version", "platform", "producer", "project_file", + "project_sha256", "runner_file", "runner_sha256", + "source_composite_algorithm", "source_composite_sha256", + "source_file_count", "source_total_bytes", "status", + ] + _exact_keys(descriptor, expected, "Parquet.jl producer descriptor") + _require(_integer_field(descriptor, "descriptor_version", "descriptor") == 1, + "producer descriptor version is not one") + _require(_string_field(descriptor, "producer", "descriptor") == "parquet-jl", + "producer descriptor has the wrong producer") + _require(_string_field(descriptor, "package_name", "descriptor") == PACKAGE_NAME, + "producer descriptor has the wrong package name") + _require(_string_field(descriptor, "package_uuid", "descriptor") == PACKAGE_UUID, + "producer descriptor has the wrong package UUID") + _require(_string_field(descriptor, "package_version", "descriptor") == + "1.0.0-DEV", "producer descriptor has the wrong package version") + _require(_string_field(descriptor, "julia_version", "descriptor") == + string(CANONICAL_JULIA_VERSION), "producer descriptor has the wrong Julia version") + _require(_string_field(descriptor, "platform", "descriptor") == CANONICAL_PLATFORM, + "producer descriptor has the wrong platform") + _require(_string_field(descriptor, "status", "descriptor") in ("planned", "verified"), + "producer descriptor has an invalid status") + _require(_string_field(descriptor, "source_composite_algorithm", "descriptor") == + SOURCE_COMPOSITE_ALGORITHM, "producer source composite algorithm differs") + _hex_sha256(descriptor["source_composite_sha256"], + "producer source composite") + _hex_sha256(descriptor["julia_executable_sha256"], "Julia executable") + _hex_sha256(descriptor["julia_runtime_tree_sha256"], "Julia runtime tree") + _require(_integer_field(descriptor, "julia_runtime_entry_count", + "descriptor") > 0, "Julia runtime entry count is not positive") + _require(_integer_field(descriptor, "julia_runtime_total_bytes", + "descriptor") > 0, "Julia runtime byte count is not positive") + _hex_sha256(descriptor["project_sha256"], "project") + _hex_sha256(descriptor["manifest_sha256"], "manifest") + _hex_sha256(descriptor["bootstrap_sha256"], "bootstrap") + _hex_sha256(descriptor["harness_sha256"], "harness") + _hex_sha256(descriptor["runner_sha256"], "runner") + _require(_integer_field(descriptor, "source_file_count", "descriptor") > 0, + "producer source file count is not positive") + _require(_integer_field(descriptor, "source_total_bytes", "descriptor") > 0, + "producer source byte count is not positive") + return +end + +function _descriptor_files(descriptor::Dict{String,Any}) + entries = _table_array(descriptor, "file", "descriptor") + files = Dict{String,String}() + for (index, entry) in enumerate(entries) + label = "descriptor file $index" + _exact_keys(entry, ["path", "sha256"], label) + relative = _string_field(entry, "path", label) + _safe_relative(relative, label) + digest = _hex_sha256(entry["sha256"], "$label digest") + _require(!haskey(files, relative), "duplicate descriptor file: $relative") + files[relative] = digest + end + _require(!isempty(files), "producer descriptor has no files") + return files +end + +function _source_inventory(root::String) + source_root = _safe_path(root, "src", "source root") + _require(isdir(source_root), "source root is absent") + entries, total = _walk_entries(source_root, MAX_SOURCE_FILES, + MAX_PINNED_FILE_BYTES, MAX_SOURCE_BYTES, "package source"; safe_links=true) + _require(all(entry -> entry.link_target === nothing, entries), + "package source contains a symbolic link") + paths = String["Project.toml"] + for entry in entries + _regular(entry.metadata) || continue + push!(paths, "src/" * entry.relative) + end + sort!(paths) + return paths, total +end + +function _source_composite(paths::Vector{String}, bytes::Dict{String,Vector{UInt8}}) + context = SHA.SHA2_256_CTX() + SHA.update!(context, codeunits(SOURCE_COMPOSITE_PREFIX)) + total = Int64(0) + for relative in sort(paths) + payload = bytes[relative] + total = Base.checked_add(total, Int64(length(payload))) + SHA.update!(context, codeunits("F\0")) + SHA.update!(context, codeunits(relative)) + SHA.update!(context, codeunits("\0")) + SHA.update!(context, codeunits(string(length(payload)))) + SHA.update!(context, codeunits("\0")) + SHA.update!(context, SHA.sha256(payload)) + end + return bytes2hex(SHA.digest!(context)), total +end + +function _load_pinned_files(root::String, descriptor::Dict{String,Any}) + expected = _descriptor_files(descriptor) + source_paths, _ = _source_inventory(root) + fixed = Dict( + "project_file" => "Project.toml", + "manifest_file" => "test/conformance/n6/julia/Manifest.toml", + "bootstrap_file" => "test/conformance/n6/julia/bootstrap.jl", + "harness_file" => "test/conformance/n6/julia/N6ParquetJLHarness.jl", + "runner_file" => "test/conformance/n6/julia/generate.jl", + ) + required = Set(source_paths) + for (field, relative) in fixed + _require(_string_field(descriptor, field, "descriptor") == relative, + "descriptor $field differs") + push!(required, relative) + end + _require(Set(keys(expected)) == required, + "producer descriptor file inventory differs from source and bootstrap inputs") + pinned = Dict{String,Vector{UInt8}}() + for relative in sort!(collect(required)) + path = _safe_path(root, relative, "descriptor file") + payload = _stable_file_bytes(path, MAX_PINNED_FILE_BYTES, + "descriptor file $relative") + _require(bytes2hex(SHA.sha256(payload)) == expected[relative], + "descriptor file digest differs: $relative") + pinned[relative] = payload + end + hash_fields = Dict( + "project_file" => "project_sha256", + "manifest_file" => "manifest_sha256", + "bootstrap_file" => "bootstrap_sha256", + "harness_file" => "harness_sha256", + "runner_file" => "runner_sha256", + ) + for (file_field, hash_field) in hash_fields + relative = descriptor[file_field] + _require(bytes2hex(SHA.sha256(pinned[relative])) == descriptor[hash_field], + "descriptor $hash_field differs from its file entry") + end + composite, total = _source_composite(source_paths, pinned) + _require(composite == descriptor["source_composite_sha256"], + "Parquet.jl source composite differs") + _require(length(source_paths) == descriptor["source_file_count"], + "Parquet.jl source file count differs") + _require(total == descriptor["source_total_bytes"], + "Parquet.jl source byte count differs") + return pinned +end + +function _validate_project(descriptor::Dict{String,Any}, pinned::Dict{String,Vector{UInt8}}) + project = TOML.parse(String(copy(pinned[descriptor["project_file"]]))) + _require(get(project, "name", nothing) == PACKAGE_NAME, + "gate project has the wrong package name") + _require(get(project, "uuid", nothing) == PACKAGE_UUID, + "gate project has the wrong package UUID") + _require(get(project, "version", nothing) == descriptor["package_version"], + "gate project has the wrong package version") + _require(!haskey(project, "preferences"), + "gate project contains exported preferences") + return project +end + +function _manifest_entries(manifest::Dict{String,Any}) + _require(get(manifest, "julia_version", nothing) == string(CANONICAL_JULIA_VERSION), + "producer manifest has the wrong Julia version") + _require(get(manifest, "manifest_format", nothing) == "2.0", + "producer manifest has the wrong format") + dependencies = get(manifest, "deps", nothing) + _require(dependencies isa AbstractDict, "producer manifest lacks dependencies") + entries = Dict{String,Dict{String,Any}}() + for (name_value, records) in dependencies + name = String(name_value) + _require(records isa AbstractVector && length(records) == 1 && + only(records) isa Dict{String,Any}, + "producer manifest dependency is not singular: $name") + entries[name] = only(records) + end + return entries +end + +function _validate_manifest(root::String, descriptor::Dict{String,Any}, + pinned::Dict{String,Vector{UInt8}}) + relative = descriptor["manifest_file"] + payload = pinned[relative] + active_path = _safe_path(root, "Manifest.toml", "active manifest") + active = _stable_file_bytes(active_path, MAX_PINNED_FILE_BYTES, + "active gate Manifest.toml") + _require(active == payload, + "active gate Manifest.toml differs from the pinned producer manifest") + manifest = TOML.parse(String(copy(payload))) + entries = _manifest_entries(manifest) + _require(haskey(entries, PACKAGE_NAME), "producer manifest lacks Parquet") + parquet = entries[PACKAGE_NAME] + _require(get(parquet, "uuid", nothing) == PACKAGE_UUID, + "producer manifest Parquet UUID differs") + _require(get(parquet, "version", nothing) == descriptor["package_version"], + "producer manifest Parquet version differs") + _require(get(parquet, "path", nothing) == ".", + "producer manifest Parquet path is not the gate root") + for (name, entry) in entries + name == PACKAGE_NAME && continue + _require(!haskey(entry, "path"), + "producer manifest has an untrusted path dependency: $name") + end + return manifest +end + +function _dependency_descriptors(descriptor::Dict{String,Any}) + records = Dict{String,Dict{String,Any}}() + for (index, entry_value) in enumerate(_table_array(descriptor, "dependency", + "descriptor")) + entry = entry_value::Dict{String,Any} + label = "descriptor dependency $index" + _exact_keys(entry, ["depot_slug", "entry_count", "git_tree_sha1", "name", + "total_bytes", "tree_sha256", "uuid", "version"], label) + name = _string_field(entry, "name", label) + _require(occursin(r"^[A-Za-z][A-Za-z0-9_]*$", name), + "$label name is invalid") + _require(!haskey(records, name), "duplicate descriptor dependency: $name") + _hex_sha1(entry["git_tree_sha1"], "$label Git tree") + _hex_sha256(entry["tree_sha256"], "$label tree") + slug = _string_field(entry, "depot_slug", label) + _require(occursin(r"^[A-Za-z0-9]{5}$", slug), + "$label depot slug is invalid") + _require(_integer_field(entry, "entry_count", label) > 0, + "$label entry count is not positive") + _require(_integer_field(entry, "total_bytes", label) >= 0, + "$label total byte count is negative") + uuid = _string_field(entry, "uuid", label) + version = _string_field(entry, "version", label) + try + Base.UUID(uuid) + VersionNumber(version) + catch error + throw(ArgumentError("$label identity is invalid: $error")) + end + records[name] = entry + end + _require(!isempty(records), "producer descriptor has no dependency trees") + return records +end + +function _validate_depot_shape(depot::String, dependencies, artifacts) + top = sort!(readdir(depot)) + _require(top == ["artifacts", "packages"], + "private depot has unexpected top-level entries") + package_root = joinpath(depot, "packages") + _require(isdir(package_root) && !islink(package_root), + "private depot packages directory is invalid") + _require(sort!(readdir(package_root)) == sort!(collect(keys(dependencies))), + "private depot package names differ from the descriptor") + for (name, entry) in dependencies + slug = Base.version_slug(Base.UUID(entry["uuid"]), + Base.SHA1(entry["git_tree_sha1"])) + _require(slug == entry["depot_slug"], + "descriptor dependency depot slug is invalid: $name") + _require(readdir(joinpath(package_root, name)) == [slug], + "private depot package slug differs: $name") + end + artifact_root = joinpath(depot, "artifacts") + _require(isdir(artifact_root) && !islink(artifact_root), + "private depot artifacts directory is invalid") + expected = sort!(String[entry["git_tree_sha1"] for entry in values(artifacts)]) + _require(sort!(readdir(artifact_root)) == expected, + "private depot artifact trees differ from the descriptor") + return +end + +function _artifact_descriptors(descriptor::Dict{String,Any}) + records = Dict{String,Dict{String,Any}}() + for (index, entry_value) in enumerate(_table_array(descriptor, "artifact", + "descriptor")) + entry = entry_value::Dict{String,Any} + label = "descriptor artifact $index" + _exact_keys(entry, ["entry_count", "git_tree_sha1", "name", "package", + "total_bytes", "tree_sha256"], label) + package = _string_field(entry, "package", label) + name = _string_field(entry, "name", label) + key = package * "\0" * name + _require(!haskey(records, key), "duplicate descriptor artifact: $package/$name") + _hex_sha1(entry["git_tree_sha1"], "$label Git tree") + _hex_sha256(entry["tree_sha256"], "$label tree") + _require(_integer_field(entry, "entry_count", label) > 0, + "$label entry count is not positive") + _require(_integer_field(entry, "total_bytes", label) >= 0, + "$label total byte count is negative") + records[key] = entry + end + _require(!isempty(records), "producer descriptor has no native artifacts") + return records +end + +function _selected_artifacts(dependency_root::String, package::String) + path = _safe_path(dependency_root, "Artifacts.toml", "$package artifact declaration") + payload = _stable_file_bytes(path, MAX_PINNED_FILE_BYTES, + "$package Artifacts.toml") + declaration = TOML.parse(String(copy(payload))) + selected = Dict{String,String}() + for (name_value, entries) in declaration + name = String(name_value) + _require(entries isa AbstractVector, + "$package artifact declaration is not an array: $name") + matches = Any[entry for entry in entries if entry isa AbstractDict && + get(entry, "arch", nothing) == "aarch64" && + get(entry, "os", nothing) == "macos"] + _require(length(matches) == 1, + "$package has no singular macOS aarch64 artifact: $name") + selected[name] = _hex_sha1(only(matches)["git-tree-sha1"], + "$package selected artifact $name") + end + _require(!isempty(selected), "$package has no selected artifacts") + return selected +end + +function _validate_dependencies(depot::String, descriptor::Dict{String,Any}, + manifest::Dict{String,Any}) + dependencies = _dependency_descriptors(descriptor) + artifacts = _artifact_descriptors(descriptor) + entries = _manifest_entries(manifest) + expected_names = Set(name for (name, entry) in entries if + haskey(entry, "git-tree-sha1")) + _require(Set(keys(dependencies)) == expected_names, + "descriptor dependency trees differ from the producer manifest") + _validate_depot_shape(depot, dependencies, artifacts) + roots = Dict{String,String}() + for (name, dependency) in dependencies + manifest_entry = entries[name] + for field in ("uuid", "version") + _require(get(manifest_entry, field, nothing) == dependency[field], + "descriptor dependency $name $field differs from the manifest") + end + _require(get(manifest_entry, "git-tree-sha1", nothing) == + dependency["git_tree_sha1"], + "descriptor dependency $name Git tree differs from the manifest") + slug = Base.version_slug(Base.UUID(dependency["uuid"]), + Base.SHA1(dependency["git_tree_sha1"])) + _require(slug == dependency["depot_slug"], + "descriptor dependency depot slug differs: $name") + root = _canonical_root(joinpath(depot, "packages", name, slug), + "private dependency $name") + identity = _tree_identity(root, MAX_TREE_ENTRIES, MAX_TREE_FILE_BYTES, + MAX_TREE_BYTES, "private dependency $name") + _require(identity.sha256 == dependency["tree_sha256"], + "private dependency tree digest differs: $name") + _require(identity.entry_count == dependency["entry_count"], + "private dependency entry count differs: $name") + _require(identity.total_bytes == dependency["total_bytes"], + "private dependency total byte count differs: $name") + roots[name] = root + end + selected = Dict{String,String}() + jll_names = sort!(String[name for name in keys(dependencies) if + endswith(name, "_jll")]) + for package in jll_names + for (name, git_tree) in _selected_artifacts(roots[package], package) + selected[package * "\0" * name] = git_tree + end + end + _require(Set(keys(artifacts)) == Set(keys(selected)), + "descriptor native artifacts differ from selected JLL artifacts") + artifact_roots = Dict{String,String}() + for (key, artifact) in artifacts + _require(artifact["git_tree_sha1"] == selected[key], + "descriptor selected artifact Git tree differs: $key") + root = _canonical_root(joinpath(depot, "artifacts", + artifact["git_tree_sha1"]), "private artifact $key") + identity = _tree_identity(root, MAX_TREE_ENTRIES, MAX_TREE_FILE_BYTES, + MAX_TREE_BYTES, "private artifact $key") + _require(identity.sha256 == artifact["tree_sha256"], + "private artifact tree digest differs: $key") + _require(identity.entry_count == artifact["entry_count"], + "private artifact entry count differs: $key") + _require(identity.total_bytes == artifact["total_bytes"], + "private artifact total byte count differs: $key") + artifact_roots[key] = root + end + return roots, artifact_roots +end + +function _validate_runtime(descriptor::Dict{String,Any}) + _require(VERSION == CANONICAL_JULIA_VERSION, + "N6 producer requires Julia $CANONICAL_JULIA_VERSION") + _require(Sys.isapple() && Sys.ARCH === :aarch64, + "N6 producer requires macOS aarch64") + command = Base.julia_cmd().exec + _require(!isempty(command), "Julia command has no executable") + executable = String(first(command)) + _require(isabspath(executable), "Julia executable is not absolute") + executable = normpath(executable) + _require(!islink(executable) && realpath(executable) == executable, + "Julia executable is not canonical") + _require(executable == joinpath(Sys.BINDIR, "julia"), + "Julia executable is outside Sys.BINDIR") + digest, _ = _stable_file_digest(executable, MAX_PINNED_FILE_BYTES, + "Julia executable") + _require(bytes2hex(digest) == descriptor["julia_executable_sha256"], + "Julia executable digest differs") + runtime_root = _canonical_root(dirname(Sys.BINDIR), "Julia runtime root") + identity = _tree_identity(runtime_root, MAX_RUNTIME_ENTRIES, + MAX_RUNTIME_FILE_BYTES, MAX_RUNTIME_BYTES, "Julia runtime"; + safe_links=false) + _require(identity.sha256 == descriptor["julia_runtime_tree_sha256"], + "Julia runtime tree digest differs") + _require(identity.entry_count == descriptor["julia_runtime_entry_count"], + "Julia runtime entry count differs") + _require(identity.total_bytes == descriptor["julia_runtime_total_bytes"], + "Julia runtime byte count differs") + return runtime_root +end + +function _package_id(name::String, entry::Dict{String,Any}) + return Base.PkgId(Base.UUID(entry["uuid"]), name) +end + +function _resolved_path(name::String, entry::Dict{String,Any}) + path = Base.locate_package(_package_id(name, entry)) + _require(path !== nothing, "manifest package cannot be resolved: $name") + absolute = normpath(String(path)) + _require(isfile(absolute) && !islink(absolute), + "resolved package entry is not a regular file: $name") + _require(realpath(absolute) == absolute, + "resolved package entry is not canonical: $name") + return absolute +end + +function _validate_resolution(context::BootstrapContext) + entries = _manifest_entries(context.manifest) + for (name, entry) in entries + path = _resolved_path(name, entry) + if name == PACKAGE_NAME + _require(path == joinpath(context.root, "src", "Parquet.jl"), + "Parquet resolves outside the read-only gate root") + elseif haskey(entry, "git-tree-sha1") + _require(_within(path, context.dependency_roots[name]), + "dependency resolves outside the private depot: $name") + else + _require(_within(path, context.runtime_root), + "stdlib resolves outside the exact Julia runtime: $name") + end + end + return +end + +function _read_descriptor(root::String) + expected = get(ENV, DESCRIPTOR_SHA256_ENV, nothing) + _hex_sha256(expected, "trusted producer descriptor") + path = _safe_path(root, DESCRIPTOR_RELATIVE, "producer descriptor") + payload = _stable_file_bytes(path, MAX_DESCRIPTOR_BYTES, + "Parquet.jl producer descriptor") + digest = bytes2hex(SHA.sha256(payload)) + _require(digest == expected, + "Parquet.jl producer descriptor digest differs from the trusted parent pin") + descriptor_value = TOML.parse(String(copy(payload))) + _require(descriptor_value isa Dict{String,Any}, + "Parquet.jl producer descriptor is not a TOML table") + descriptor = descriptor_value::Dict{String,Any} + _validate_descriptor(descriptor) + return path, digest, descriptor +end + +function _verify_preload() + _validate_bootstrap_stdlibs() + root = _canonical_root(pwd(), "N6 gate root") + depot_value = get(ENV, "JULIA_DEPOT_PATH", nothing) + _require(depot_value isa String && !isempty(depot_value), + "JULIA_DEPOT_PATH is absent") + _require(!occursin(':', depot_value), + "JULIA_DEPOT_PATH contains more than one depot") + depot = _canonical_root(depot_value, "N6 private depot") + _require(!_within(depot, root) && !_within(root, depot), + "gate root and private depot overlap") + _validate_environment(root, depot) + expected_program = joinpath(root, "test", "conformance", "n6", "julia", + "bootstrap.jl") + _require(!isempty(PROGRAM_FILE) && realpath(PROGRAM_FILE) == expected_program, + "Julia did not execute the pinned N6 bootstrap file directly") + descriptor_path, descriptor_sha256, descriptor = _read_descriptor(root) + _assert_readonly_gate(root) + _assert_readonly_depot(depot) + pinned = _load_pinned_files(root, descriptor) + _validate_project(descriptor, pinned) + manifest = _validate_manifest(root, descriptor, pinned) + runtime_root = _validate_runtime(descriptor) + dependency_roots, artifact_roots = _validate_dependencies(depot, descriptor, + manifest) + context = BootstrapContext(root, depot, runtime_root, descriptor_path, + descriptor_sha256, descriptor, manifest, pinned, dependency_roots, + artifact_roots) + _validate_resolution(context) + _require(bytes2hex(SHA.sha256(_stable_file_bytes(descriptor_path, + MAX_DESCRIPTOR_BYTES, "Parquet.jl producer descriptor"))) == + descriptor_sha256, "producer descriptor changed before package loading") + return context +end + +function _verify_tree_identities(context::BootstrapContext) + runtime = _tree_identity(context.runtime_root, MAX_RUNTIME_ENTRIES, + MAX_RUNTIME_FILE_BYTES, MAX_RUNTIME_BYTES, "Julia runtime"; + safe_links=false) + _require(runtime.sha256 == context.descriptor["julia_runtime_tree_sha256"], + "Julia runtime changed after package execution") + _require(runtime.entry_count == + context.descriptor["julia_runtime_entry_count"], + "Julia runtime entry count changed after package execution") + _require(runtime.total_bytes == + context.descriptor["julia_runtime_total_bytes"], + "Julia runtime byte count changed after package execution") + dependencies = _dependency_descriptors(context.descriptor) + for (name, root) in context.dependency_roots + identity = _tree_identity(root, MAX_TREE_ENTRIES, MAX_TREE_FILE_BYTES, + MAX_TREE_BYTES, "private dependency $name") + _require(identity.sha256 == dependencies[name]["tree_sha256"], + "private dependency changed after package execution: $name") + _require(identity.entry_count == dependencies[name]["entry_count"], + "private dependency entry count changed after package execution: $name") + _require(identity.total_bytes == dependencies[name]["total_bytes"], + "private dependency byte count changed after package execution: $name") + end + artifacts = _artifact_descriptors(context.descriptor) + for (key, root) in context.artifact_roots + identity = _tree_identity(root, MAX_TREE_ENTRIES, MAX_TREE_FILE_BYTES, + MAX_TREE_BYTES, "private artifact $key") + _require(identity.sha256 == artifacts[key]["tree_sha256"], + "private artifact changed after package execution: $key") + _require(identity.entry_count == artifacts[key]["entry_count"], + "private artifact entry count changed after package execution: $key") + _require(identity.total_bytes == artifacts[key]["total_bytes"], + "private artifact byte count changed after package execution: $key") + end + return +end + +function _verify_postload(context::BootstrapContext) + _validate_environment(context.root, context.depot) + _validate_resolution(context) + _require(pathof(Parquet) == joinpath(context.root, "src", "Parquet.jl"), + "loaded Parquet module is outside the read-only gate root") + descriptor_payload = _stable_file_bytes(context.descriptor_path, + MAX_DESCRIPTOR_BYTES, "Parquet.jl producer descriptor") + _require(bytes2hex(SHA.sha256(descriptor_payload)) == context.descriptor_sha256, + "producer descriptor changed after package execution") + pinned = _load_pinned_files(context.root, context.descriptor) + _require(pinned == context.pinned_bytes, + "descriptor-bound input changed after package execution") + _verify_tree_identities(context) + _assert_readonly_gate(context.root) + _assert_readonly_depot(context.depot) + return +end + +const CONTEXT = _verify_preload() + +using Parquet + +_validate_resolution(CONTEXT) +_require(pathof(Parquet) == joinpath(CONTEXT.root, "src", "Parquet.jl"), + "loaded Parquet module is outside the read-only gate root") + +const HARNESS_RELATIVE = CONTEXT.descriptor["harness_file"] +Base.include_string(@__MODULE__, String(copy(CONTEXT.pinned_bytes[HARNESS_RELATIVE])), + joinpath(CONTEXT.root, split(HARNESS_RELATIVE, '/')...)) +const OUTPUT = N6ParquetJLHarness.runharness(:check) + +_verify_postload(CONTEXT) +println("checked ", length(OUTPUT.files), + " generated N6 files and 96 evidence records in the isolated Parquet.jl gate") + +end diff --git a/test/conformance/n6/julia/generate.jl b/test/conformance/n6/julia/generate.jl new file mode 100644 index 0000000..adb9867 --- /dev/null +++ b/test/conformance/n6/julia/generate.jl @@ -0,0 +1,13 @@ +using Parquet + +include(joinpath(@__DIR__, "N6ParquetJLHarness.jl")) + +length(ARGS) == 1 || throw(ArgumentError( + "usage: generate.jl --write|--check")) +mode = ARGS[1] == "--write" ? :write : + ARGS[1] == "--check" ? :check : throw(ArgumentError( + "usage: generate.jl --write|--check")) +output = N6ParquetJLHarness.runharness(mode) +verb = mode === :write ? "wrote" : "checked" +println(verb, " ", length(output.files), + " generated N6 files and 96 evidence records") diff --git a/test/conformance/n6/julia/parquet-jl-producer.toml b/test/conformance/n6/julia/parquet-jl-producer.toml new file mode 100644 index 0000000..c2a2c01 --- /dev/null +++ b/test/conformance/n6/julia/parquet-jl-producer.toml @@ -0,0 +1,416 @@ +descriptor_version = 1 +status = "verified" +producer = "parquet-jl" +package_name = "Parquet" +package_uuid = "626c502c-15b0-58ad-a749-f091afb673ae" +package_version = "1.0.0-DEV" +julia_version = "1.12.6" +platform = "macos-15-arm64" +julia_executable_sha256 = "9ad38bea81ecace044a4bdef2a0246dee94cb8a44c9420809cc00f9872651c64" +julia_runtime_tree_sha256 = "273ec71de498a36c77a7e4bb3af4a3f75c338bd1cfe255cab30805b6a2cda76e" +julia_runtime_entry_count = 7180 +julia_runtime_total_bytes = 820750312 +source_composite_algorithm = "parquet-jl-n6-source-composite-v1" +source_composite_sha256 = "ea75000a8b4505c73efe50476a45dfe427ed5c8f7123fbaf244390f2b26b0c80" +source_file_count = 38 +source_total_bytes = 1160815 +project_file = "Project.toml" +project_sha256 = "267fbe0fb90440d6de6bf855a971121f665360cd9f7d256a0d9f0d039e2e424a" +manifest_file = "test/conformance/n6/julia/Manifest.toml" +manifest_sha256 = "186cc635d7fcbc19f9acabc0ba9198d837d6eb0fccdd1a431a3137a6a43be565" +harness_file = "test/conformance/n6/julia/N6ParquetJLHarness.jl" +harness_sha256 = "e3e0e903e3ecca103eeb5ec2d11aa0ee81c28f4344f48c8bfd1df0eaa5c01370" +runner_file = "test/conformance/n6/julia/generate.jl" +runner_sha256 = "508a7d640a7f24e6548f2973f6e2aa867834d1f91dd98b577ef40417d3209dd4" +bootstrap_file = "test/conformance/n6/julia/bootstrap.jl" +bootstrap_sha256 = "77dbc466d90fdc9e5795e33e2323dd6520696084849664519590c0e1d4af2857" + +[[artifact]] +package = "Lz4_jll" +name = "Lz4" +git_tree_sha1 = "2c9ca887239bdb5da1d070c82a34c291024b64d5" +tree_sha256 = "b50c12bc05604ab41a79c3eae486d981dc58d69c533792e4b4ad1f0fa047cae3" +entry_count = 28 +total_bytes = 810564 + +[[artifact]] +package = "Zstd_jll" +name = "Zstd" +git_tree_sha1 = "8da603395acfbdbef8c5de3b7223aeb9276ecbdb" +tree_sha256 = "3954c59a164b07234ee25b728322a8e1a2809ebdb8433fca14650e594c9df8c9" +entry_count = 30 +total_bytes = 1451985 + +[[artifact]] +package = "brotli_jll" +name = "brotli" +git_tree_sha1 = "674b42b4725e390d6209dd83f6a1d77dcbf6a6ca" +tree_sha256 = "d211a457a19601aeceb318e2ed129584461bc4d857e504fc80a1dcf00fa9d689" +entry_count = 35 +total_bytes = 1208583 + +[[artifact]] +package = "snappy_jll" +name = "snappy" +git_tree_sha1 = "6f71d2334e4b4c745ce16a4d05e88c5e593bfa11" +tree_sha256 = "6e9b2b48c289aa7a1a4cabb6920cfae5e7965288ef39ff63feb39ec4ccfeae5b" +entry_count = 19 +total_bytes = 117253 + +[[dependency]] +name = "CRC32" +uuid = "b4567568-9dcc-467e-9b62-c342d3a501d3" +version = "1.1.0" +git_tree_sha1 = "253002ec391e61dadb453d922f2d278459b0bb0f" +depot_slug = "xHood" +tree_sha256 = "9486a20eb96a01cce38d13d6ea083325de53a5c270853cc7b30942540bf4561e" +entry_count = 14 +total_bytes = 18752 + +[[dependency]] +name = "ChunkCodecCore" +uuid = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" +version = "1.0.1" +git_tree_sha1 = "1a3ad7e16a321667698a19e77362b35a1e94c544" +depot_slug = "4xQYG" +tree_sha256 = "4ed9f94b8fce0150461696bebd0a9ebe7e40ee23b31b812aca52bbe6591d7378" +entry_count = 14 +total_bytes = 35499 + +[[dependency]] +name = "ChunkCodecLibBrotli" +uuid = "653b0ff7-85b5-4442-93c1-dcc330d3ec7d" +version = "1.0.0" +git_tree_sha1 = "45709ad3ba09bdff5e6481d2c1727b1499989997" +depot_slug = "uLacM" +tree_sha256 = "a8b4a4e32bdbc6d51709056c0ab2cccbdd28e4a8b1eb92b590317c2af2a623be" +entry_count = 13 +total_bytes = 25837 + +[[dependency]] +name = "ChunkCodecLibLz4" +uuid = "7e9cc85e-5614-42a3-ad86-b78f920b38a5" +version = "1.0.0" +git_tree_sha1 = "0a4d7695ef98ab714efe5aef26fc35c3b0b4c1ee" +depot_slug = "6YUUc" +tree_sha256 = "18ea7b29582809a2cfbd163a63d1014161db79378c9ee914a29b83dd6431852c" +entry_count = 18 +total_bytes = 75408 + +[[dependency]] +name = "ChunkCodecLibSnappy" +uuid = "eac87354-86d5-4a5b-ab5f-a6ee56b239b3" +version = "1.0.0" +git_tree_sha1 = "a9e98b8cc7ccdcfcb406773a6c58987daa6eda05" +depot_slug = "KH1tp" +tree_sha256 = "8f914459e1b87f7914174cf5318e252c889d3fc8da8860c931700c8a1c15302d" +entry_count = 12 +total_bytes = 11447 + +[[dependency]] +name = "ChunkCodecLibZlib" +uuid = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" +version = "1.1.0" +git_tree_sha1 = "d4101e848e8d3f585d61d244c2fe0c80a70e6b3b" +depot_slug = "EveAQ" +tree_sha256 = "ea0c59fe49692daa03ab0c37655029351ff2c307c7158881cdc8de7682a3571a" +entry_count = 13 +total_bytes = 34112 + +[[dependency]] +name = "ChunkCodecLibZstd" +uuid = "55437552-ac27-4d47-9aa3-63184e8fd398" +version = "1.0.0" +git_tree_sha1 = "34d9873079e4cb3d0c62926a225136824677073f" +depot_slug = "Rfg35" +tree_sha256 = "cc87b948c86575ef22349f72173266516784e50ef7f895d50163b38a1997b2f1" +entry_count = 12 +total_bytes = 26757 + +[[dependency]] +name = "DataAPI" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" +git_tree_sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +depot_slug = "atdEM" +tree_sha256 = "f3e70c0243132be0a85d842ab19750fa8ce389096163f4dc88434579e13c0fda" +entry_count = 12 +total_bytes = 38853 + +[[dependency]] +name = "DataValueInterfaces" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" +git_tree_sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +depot_slug = "0j6Kp" +tree_sha256 = "b55f96545d799a98bf7551240eeba5b502014009b93da1012543d2859cc14cde" +entry_count = 9 +total_bytes = 4922 + +[[dependency]] +name = "IteratorInterfaceExtensions" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" +git_tree_sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +depot_slug = "NZdaj" +tree_sha256 = "1c3f117b5d5904caeb9c9a5f8c6754b792f922d6d88a5ced1dfeb96fcf84373d" +entry_count = 17 +total_bytes = 8492 + +[[dependency]] +name = "JLLWrappers" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.8.0" +git_tree_sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" +depot_slug = "Kp7TC" +tree_sha256 = "11eb9155ce8efd84e9893cb49f3f94e4dc457159a833b25717a74461c73fe1a2" +entry_count = 23 +total_bytes = 38958 + +[[dependency]] +name = "Lz4_jll" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.10.1+0" +git_tree_sha1 = "191686b1ac1ea9c89fc52e996ad15d1d241d1e33" +depot_slug = "l1tka" +tree_sha256 = "66db364f1f17154d0ba9443b0506817bb354ba4df00f953cefa00ac057b9913a" +entry_count = 26 +total_bytes = 28339 + +[[dependency]] +name = "OrderedCollections" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "2.0.1" +git_tree_sha1 = "05f45c2e0de6259db764adbfd2f1dc6d3f8de13c" +depot_slug = "1sijc" +tree_sha256 = "057112090b0c9569f6e7acaaa3f251ba56ef0dfdbc535af2b7304e88fc5c9b49" +entry_count = 32 +total_bytes = 107628 + +[[dependency]] +name = "Preferences" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" +git_tree_sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +depot_slug = "kUJxq" +tree_sha256 = "4e0d08fd3b2f5986a45ed4e11d40b74626b13242156528f228fa02e795e5f700" +entry_count = 54 +total_bytes = 69993 + +[[dependency]] +name = "TableTraits" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" +git_tree_sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +depot_slug = "o8VMV" +tree_sha256 = "13e2c000ad9cfc736bf872f29a5fd1418d3dc5ffa39098f0c7ac96ca29fa46fe" +entry_count = 25 +total_bytes = 9660 + +[[dependency]] +name = "Tables" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.13.0" +git_tree_sha1 = "0f38a06c83f0007bbab3cf911262841c9a0f07e0" +depot_slug = "oWklV" +tree_sha256 = "1dce436b2726b4384e626da0d053d46becc25bafbd6e2d673f925d5e2b94c2bf" +entry_count = 31 +total_bytes = 167977 + +[[dependency]] +name = "Zstd_jll" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" +git_tree_sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" +depot_slug = "YX1LN" +tree_sha256 = "97f416a13b4a8b78b9a064a11e9dcd11450de4de1e48a802bce73987e99706e9" +entry_count = 26 +total_bytes = 23837 + +[[dependency]] +name = "brotli_jll" +uuid = "4611771a-a7d2-5e23-8d00-b1becdba1aae" +version = "1.2.0+0" +git_tree_sha1 = "46fda47f4215c957bc92fd5fbb5ad04fee1e3743" +depot_slug = "i5MrA" +tree_sha256 = "6057f17fefd89a28977664969104fd22df644293524e24c4cf32136b51254bd1" +entry_count = 26 +total_bytes = 30198 + +[[dependency]] +name = "snappy_jll" +uuid = "fe1e1685-f7be-5f59-ac9f-4ca204017dfd" +version = "1.2.3+0" +git_tree_sha1 = "ca88363dd41d2547f52118287dd34dbbc14f3eb7" +depot_slug = "GeeRK" +tree_sha256 = "401c2cf971083bfd2dc32633a8ed61464620ab990599d0ba48ce4d767ca69509" +entry_count = 40 +total_bytes = 32853 + +[[file]] +path = "Project.toml" +sha256 = "267fbe0fb90440d6de6bf855a971121f665360cd9f7d256a0d9f0d039e2e424a" + +[[file]] +path = "src/Parquet.jl" +sha256 = "33aaba84985447f211fdcd4ea2d50a47d627356474c2180a734a55cf7d93caa9" + +[[file]] +path = "src/bss.jl" +sha256 = "15e56e76b50c3858bf2b34d2fdf8f2f24d23ba0b373b144e3c855b347c957ccd" + +[[file]] +path = "src/checksum.jl" +sha256 = "94d68fb49a163013c954f53282164e088624875a0f689d402c40cbc496667e61" + +[[file]] +path = "src/codecs.jl" +sha256 = "190441af4a717b25154a45b83dd8418197803442a8d9ca010ae172a1bb30e56d" + +[[file]] +path = "src/column.jl" +sha256 = "8e7c2c87c3e042027ce679caf2b8e11a8e04821fbc501ddd2d6b5419ffa0d93f" + +[[file]] +path = "src/delta.jl" +sha256 = "0a15f210f3e666c89e617432cca6f87369cf1443865db7fc0fcbf37db5805c98" + +[[file]] +path = "src/dictionary.jl" +sha256 = "e3082dfcd3317b0dd03c7cea4f99f0860168fe81a3fedbcc4ecf01c68676aef1" + +[[file]] +path = "src/dremel.jl" +sha256 = "87c17664a23e3a8d2cdf5bd734b56ed6a78012d957ca4d24b439b8792e25d91a" + +[[file]] +path = "src/errors.jl" +sha256 = "5b34d5aa45a1963a73ed4966a12f5e06f843eebd5af35b4a5603beb18910c9b2" + +[[file]] +path = "src/footer.jl" +sha256 = "99873f8ae59e0af45219174077704cb28c860a22b681c06b85415d3f35d014bc" + +[[file]] +path = "src/logical.jl" +sha256 = "713875346a5295b38d6c0b68ed165d79f357b156f9e3a2f8c6632526618b224c" + +[[file]] +path = "src/logical_binary.jl" +sha256 = "fb181b2a7c52e6877910c91c1a90ed226307fe50fc872914f0ff624735726fad" + +[[file]] +path = "src/logical_bson.jl" +sha256 = "619fc9e7381f4dbd5347bc11bdbdc0d1ab7530ba6f1e4a8599a4b78cbd1aafac" + +[[file]] +path = "src/logical_column.jl" +sha256 = "bacd73f9cbc0981cef3475182ba2a200c4a8ec8063a6691fe06228a184b91c6a" + +[[file]] +path = "src/logical_decimal.jl" +sha256 = "b59f3b6a76ef4e7ee3e390afa2fa0a6e091d3ae6541067881a6e785ff693e715" + +[[file]] +path = "src/logical_json.jl" +sha256 = "ac01cb15a9b7dddaeae4efa36353ec79d8c908e84c2fed31dc4998fb3f571420" + +[[file]] +path = "src/logical_temporal.jl" +sha256 = "8f330dc6f7ad1087ed619fb324fe436ace69fa4e0164f9d6312365feda19b20e" + +[[file]] +path = "src/metadata/parquet.jl" +sha256 = "c73bff99ea97b6087f950c84b68186df6187cbdeb8221b915b272d1bd6e75607" + +[[file]] +path = "src/nested_reader.jl" +sha256 = "6e66c017ba29729897b46d2bacaaa75e2ffc52857223d3b6c8c1d20e7cd6c824" + +[[file]] +path = "src/nested_schema.jl" +sha256 = "a3d0948612261fad808567b0b698d637a77f6b86aece6e8405e845383bbcb11b" + +[[file]] +path = "src/nested_table.jl" +sha256 = "a05116b7d9f140121b8820a48235e624b872bbae3561abde641f6511f6485284" + +[[file]] +path = "src/page.jl" +sha256 = "1cc74749c063e56705ab24e9711671614eedf191cb902be887a2dbc141004227" + +[[file]] +path = "src/page_index.jl" +sha256 = "94d9d292b0c50419b7a3e1602afc234065d65c8182ca6e9ab6aae81a6f0d671d" + +[[file]] +path = "src/plain.jl" +sha256 = "696e44d26e85eedf37861d435c9f007e7a64085fe2ac2e44faca034f2432580d" + +[[file]] +path = "src/rle.jl" +sha256 = "31d565e400228f663e58389a9d8eeff42d4cd24920cfa104872f95edf98761af" + +[[file]] +path = "src/schema.jl" +sha256 = "d111b0cc0e1152002506a39b9ea8ddcd7ed1ead0ded3a185f25afe882b1093f6" + +[[file]] +path = "src/source.jl" +sha256 = "7f599b091a1e12d8e2c7c887bccb217e6e8dfb390fa08b152750b20cf1c492be" + +[[file]] +path = "src/statistics.jl" +sha256 = "85b530c96ee8d52f806e225f55692a1aff852eb2238bb135e0632548468f3e58" + +[[file]] +path = "src/table.jl" +sha256 = "2ba2989328a546fb48f17dc9cd3a65b836ad250a6b01ba403b930449939b8511" + +[[file]] +path = "src/thrift.jl" +sha256 = "ef6cc6baf184886f0cd51a14fa81c7eb12a92190d3d91b5f0882ed43391a0071" + +[[file]] +path = "src/vectors.jl" +sha256 = "6af4fa94b6633f33a45f50af8560cd6c9bde3213f8c89c6009a0d04e3c1a2e10" + +[[file]] +path = "src/write.jl" +sha256 = "79ed219be9d3d832af56e67b23535b48bb361c628fecf9f99f977495adca5490" + +[[file]] +path = "src/write_logical.jl" +sha256 = "220757c98e62562f6654bf7fadcb19253c44b0318217b66c9753811b1bf63736" + +[[file]] +path = "src/write_nested.jl" +sha256 = "a15ac92645a11bf658d02d0b9488be169dfa25beb7ac3281b63fbbf7946f3e04" + +[[file]] +path = "src/write_provenance.jl" +sha256 = "78f767df87ff41861440a9601bb8567ee5b3d3e23c8b90a07ce1c6fac3509aa6" + +[[file]] +path = "src/write_splitting.jl" +sha256 = "8f74e33c0b2c696ca2aec7916fa583535092c35f9e488ed213c9b53218bf1b3e" + +[[file]] +path = "src/write_statistics.jl" +sha256 = "69edff59feae7b9b05f8e4b14c7eee7dd3e3482e40be278f8bfb904dedb3b510" + +[[file]] +path = "test/conformance/n6/julia/Manifest.toml" +sha256 = "186cc635d7fcbc19f9acabc0ba9198d837d6eb0fccdd1a431a3137a6a43be565" + +[[file]] +path = "test/conformance/n6/julia/N6ParquetJLHarness.jl" +sha256 = "e3e0e903e3ecca103eeb5ec2d11aa0ee81c28f4344f48c8bfd1df0eaa5c01370" + +[[file]] +path = "test/conformance/n6/julia/bootstrap.jl" +sha256 = "77dbc466d90fdc9e5795e33e2323dd6520696084849664519590c0e1d4af2857" + +[[file]] +path = "test/conformance/n6/julia/generate.jl" +sha256 = "508a7d640a7f24e6548f2973f6e2aa867834d1f91dd98b577ef40417d3209dd4" diff --git a/test/conformance/n6/julia/producer_gate.jl b/test/conformance/n6/julia/producer_gate.jl new file mode 100644 index 0000000..fb3ef12 --- /dev/null +++ b/test/conformance/n6/julia/producer_gate.jl @@ -0,0 +1,616 @@ +using Pkg +using UUIDs + +const PARQUET_JL_DESCRIPTOR_RELATIVE = + "test/conformance/n6/julia/parquet-jl-producer.toml" +const PARQUET_JL_MANIFEST_RELATIVE = + "test/conformance/n6/julia/Manifest.toml" +const PARQUET_JL_BOOTSTRAP_RELATIVE = + "test/conformance/n6/julia/bootstrap.jl" +const PARQUET_JL_SOURCE_COMPOSITE_ALGORITHM = + "parquet-jl-n6-source-composite-v1" +const PARQUET_JL_SOURCE_COMPOSITE_PREFIX = + UInt8[codeunits(PARQUET_JL_SOURCE_COMPOSITE_ALGORITHM * "\0")...] +const PARQUET_JL_DEPOT_SOURCE_ENV = "PARQUET_N6_JULIA_SOURCE_DEPOT" +const PARQUET_JL_TREE_ENTRY_LIMIT = 10_000 +const PARQUET_JL_TREE_FILE_LIMIT = 256 * 1024 * 1024 +const PARQUET_JL_TREE_TOTAL_LIMIT = 512 * 1024 * 1024 + +function parquet_jl_source_paths(root::AbstractString) + project = joinpath(root, "Project.toml") + islink(project) && error("Parquet.jl Project.toml is a symbolic link") + isfile(project) || error("Parquet.jl Project.toml is absent") + source = joinpath(root, "src") + islink(source) && error("Parquet.jl source root is a symbolic link") + isdir(source) || error("Parquet.jl source root is absent") + paths = String["Project.toml"] + for (directory, directories, files) in walkdir(source; follow_symlinks=false) + for name in directories + path = joinpath(directory, name) + islink(path) && error("Parquet.jl source directory is a symbolic link: " * + replace(relpath(path, root), Base.Filesystem.path_separator => '/')) + end + for name in files + path = joinpath(directory, name) + relative = replace(relpath(path, root), + Base.Filesystem.path_separator => '/') + islink(path) && error("Parquet.jl source file is a symbolic link: $relative") + isfile(path) || error("Parquet.jl source entry is not a file: $relative") + safe_relative(relative) || error("unsafe Parquet.jl source path: $relative") + push!(paths, relative) + end + end + sort!(paths) + return paths +end + +function parquet_jl_source_composite(entries, snapshots) + rows = Pair{String,FileSnapshot}[] + for item in entries + path = item["path"] + snapshot = control_snapshot(snapshots, path) + snapshot.sha256 == item["sha256"] || + error("Parquet.jl source file hash differs: $path") + push!(rows, path => snapshot) + end + sort!(rows; by=first) + paths = first.(rows) + length(paths) == length(unique(paths)) || + error("Parquet.jl source descriptor has duplicate paths") + output = IOBuffer() + write(output, PARQUET_JL_SOURCE_COMPOSITE_PREFIX) + for (path, snapshot) in rows + safe_relative(path) || error("unsafe Parquet.jl composite path: $path") + all(isascii, path) || error("non-ASCII Parquet.jl composite path: $path") + write(output, "F\0", path, '\0', string(length(snapshot.payload)), '\0') + write(output, hex2bytes(snapshot.sha256)) + end + return bytes2hex(SHA.sha256(take!(output))) +end + +function parquet_jl_manifest_dependencies(manifest) + dependencies = Dict{String,Dict{String,Any}}() + path_packages = String[] + for (name, records) in manifest["deps"] + length(records) == 1 || error("Julia manifest has duplicate package $name") + record = only(records) + if haskey(record, "path") + push!(path_packages, name) + continue + end + haskey(record, "git-tree-sha1") || continue + dependencies[name] = record + end + path_packages == ["Parquet"] || + error("Julia manifest path package inventory differs") + only(manifest["deps"]["Parquet"])["path"] == "." || + error("Julia manifest Parquet path differs") + return dependencies +end + +function parquet_jl_dependency_map(descriptor) + dependencies = Dict{String,Dict{String,Any}}() + for dependency in descriptor["dependency"] + name = dependency["name"] + haskey(dependencies, name) && + error("duplicate Parquet.jl dependency descriptor: $name") + dependencies[name] = dependency + end + return dependencies +end + +function validate_parquet_jl_dependency_descriptor(dependency, manifest_entry) + require_keys(dependency, ["name", "uuid", "version", "git_tree_sha1", + "depot_slug", "tree_sha256", "entry_count", "total_bytes"]) + @test all(field -> dependency[field] isa String, + ("name", "uuid", "version", "git_tree_sha1", "depot_slug", + "tree_sha256")) + @test dependency["uuid"] == manifest_entry["uuid"] + @test dependency["version"] == manifest_entry["version"] + @test dependency["git_tree_sha1"] == manifest_entry["git-tree-sha1"] + @test occursin(GIT_PATTERN, dependency["git_tree_sha1"]) + @test occursin(SHA256_PATTERN, dependency["tree_sha256"]) + @test occursin(r"^[A-Za-z0-9]{5}$", dependency["depot_slug"]) + @test dependency["entry_count"] isa Int64 + @test dependency["total_bytes"] isa Int64 + @test 0 < dependency["entry_count"] <= PARQUET_JL_TREE_ENTRY_LIMIT + @test 0 < dependency["total_bytes"] <= PARQUET_JL_TREE_TOTAL_LIMIT + uuid = UUID(dependency["uuid"]) + tree = Base.SHA1(hex2bytes(dependency["git_tree_sha1"])) + @test dependency["depot_slug"] == Base.version_slug(uuid, tree) + return +end + +function validate_parquet_jl_artifact_descriptor(artifact) + require_keys(artifact, ["package", "name", "git_tree_sha1", + "tree_sha256", "entry_count", "total_bytes"]) + @test all(field -> artifact[field] isa String, + ("package", "name", "git_tree_sha1", "tree_sha256")) + @test occursin(GIT_PATTERN, artifact["git_tree_sha1"]) + @test occursin(SHA256_PATTERN, artifact["tree_sha256"]) + @test artifact["entry_count"] isa Int64 + @test artifact["total_bytes"] isa Int64 + @test 0 < artifact["entry_count"] <= PARQUET_JL_TREE_ENTRY_LIMIT + @test 0 < artifact["total_bytes"] <= PARQUET_JL_TREE_TOTAL_LIMIT + return +end + +function validate_parquet_jl_descriptor(manifest, capabilities, snapshots) + descriptor_snapshot = control_snapshot(snapshots, + manifest["parquet_jl_producer_descriptor_file"]) + @test descriptor_snapshot.sha256 == + manifest["parquet_jl_producer_descriptor_sha256"] + descriptor = parse_toml_snapshot(descriptor_snapshot) + require_keys(descriptor, ["descriptor_version", "status", "producer", + "package_name", "package_uuid", "package_version", "julia_version", + "platform", + "julia_executable_sha256", "julia_runtime_tree_sha256", + "julia_runtime_entry_count", "julia_runtime_total_bytes", + "source_composite_algorithm", "source_composite_sha256", + "source_file_count", "source_total_bytes", "project_file", + "project_sha256", "manifest_file", "manifest_sha256", + "harness_file", "harness_sha256", "runner_file", "runner_sha256", + "bootstrap_file", "bootstrap_sha256", "artifact", "dependency", + "file"]) + @test descriptor["descriptor_version"] == 1 + @test descriptor["status"] in ("planned", "verified") + @test descriptor["producer"] == "parquet-jl" + @test descriptor["package_name"] == "Parquet" + @test descriptor["package_uuid"] == + "626c502c-15b0-58ad-a749-f091afb673ae" + @test descriptor["package_version"] == "1.0.0-DEV" + @test descriptor["julia_version"] == "1.12.6" + @test descriptor["platform"] == "macos-15-arm64" + @test descriptor["julia_runtime_entry_count"] == 7180 + @test descriptor["julia_runtime_total_bytes"] == 820750312 + @test descriptor["source_composite_algorithm"] == + PARQUET_JL_SOURCE_COMPOSITE_ALGORITHM + for (file_key, hash_key) in (("project_file", "project_sha256"), + ("manifest_file", "manifest_sha256"), + ("harness_file", "harness_sha256"), + ("runner_file", "runner_sha256"), + ("bootstrap_file", "bootstrap_sha256")) + @test safe_relative(descriptor[file_key]) + @test occursin(SHA256_PATTERN, descriptor[hash_key]) + @test control_snapshot(snapshots, + descriptor[file_key]).sha256 == descriptor[hash_key] + end + @test descriptor["project_file"] == "Project.toml" + @test descriptor["manifest_file"] == PARQUET_JL_MANIFEST_RELATIVE + @test descriptor["bootstrap_file"] == PARQUET_JL_BOOTSTRAP_RELATIVE + file_map = Dict{String,Dict{String,Any}}() + for item in descriptor["file"] + require_keys(item, ["path", "sha256"]) + @test item["path"] isa String + @test item["sha256"] isa String + @test safe_relative(item["path"]) + @test occursin(SHA256_PATTERN, item["sha256"]) + @test !haskey(file_map, item["path"]) + @test control_snapshot(snapshots, item["path"]).sha256 == item["sha256"] + file_map[item["path"]] = item + end + source_paths = parquet_jl_source_paths(REPO_DIR) + source_entries = Dict{String,Any}[file_map[path] for path in source_paths] + support_paths = Set([descriptor["manifest_file"], descriptor["harness_file"], + descriptor["runner_file"], descriptor["bootstrap_file"]]) + @test Set(keys(file_map)) == union(Set(source_paths), support_paths) + @test descriptor["source_file_count"] == length(source_paths) + @test descriptor["source_total_bytes"] == + sum(length(control_snapshot(snapshots, path).payload) for path in source_paths) + composite = parquet_jl_source_composite(source_entries, snapshots) + @test composite == descriptor["source_composite_sha256"] + @test composite == manifest["parquet_jl_source_composite_sha256"] + julia_manifest = TOML.parse(String(copy(control_snapshot(snapshots, + descriptor["manifest_file"]).payload))) + @test julia_manifest["julia_version"] == descriptor["julia_version"] + manifest_dependencies = parquet_jl_manifest_dependencies(julia_manifest) + descriptor_dependencies = parquet_jl_dependency_map(descriptor) + @test Set(keys(descriptor_dependencies)) == Set(keys(manifest_dependencies)) + for (name, dependency) in descriptor_dependencies + validate_parquet_jl_dependency_descriptor(dependency, + manifest_dependencies[name]) + end + artifact_keys = Set{Tuple{String,String}}() + for artifact in descriptor["artifact"] + validate_parquet_jl_artifact_descriptor(artifact) + key = (artifact["package"], artifact["name"]) + @test key ∉ artifact_keys + push!(artifact_keys, key) + end + @test artifact_keys == Set([ + ("Lz4_jll", "Lz4"), + ("Zstd_jll", "Zstd"), + ("brotli_jll", "brotli"), + ("snappy_jll", "snappy"), + ]) + authority = only(filter(item -> item["id"] == "parquet-jl", + capabilities["authority"])) + @test authority["revision"] == descriptor["source_composite_sha256"] + @test descriptor_snapshot.sha256 in authority["toolchain_sha256"] + toolchain = only(filter(item -> item["id"] == "parquet-jl-evidence", + manifest["toolchain"])) + @test toolchain["status"] == descriptor["status"] + @test toolchain_artifact(manifest, "parquet-jl-evidence", + "parquet-jl-producer.toml") == descriptor_snapshot.sha256 + return descriptor +end + +function parquet_jl_tree_identity(root::AbstractString, item) + inventory = validate_bounded_tree(root; + max_entries=Int(item["entry_count"]), + max_file_bytes=min(Int(item["total_bytes"]), + PARQUET_JL_TREE_FILE_LIMIT), + max_total_bytes=Int(item["total_bytes"])) + require_gate(inventory.entries == item["entry_count"], + "tree entry count differs: $root") + require_gate(inventory.bytes == item["total_bytes"], + "tree byte count differs: $root") + require_gate(inventory.sha256 == item["tree_sha256"], + "tree SHA-256 differs: $root") + require_gate(bytes2hex(Pkg.GitTools.tree_hash(root)) == + item["git_tree_sha1"], "tree Git identity differs: $root") + return inventory +end + +function parquet_jl_source_depots() + explicit = get(ENV, PARQUET_JL_DEPOT_SOURCE_ENV, "") + if !isempty(explicit) + islink(explicit) && error("Parquet.jl source depot is a symbolic link") + isdir(explicit) || error("Parquet.jl source depot is absent") + return String[realpath(explicit)] + end + roots = String[] + for depot in Base.DEPOT_PATH + isdir(depot) || continue + root = realpath(depot) + root in roots || push!(roots, root) + end + isempty(roots) && error("no Julia source depot is available") + return roots +end + +function locate_parquet_jl_dependency(item, depots) + relative = joinpath("packages", item["name"], item["depot_slug"]) + failures = String[] + for depot in depots + candidate = joinpath(depot, relative) + ispath(candidate) || continue + try + islink(candidate) && error("dependency root is a symbolic link") + parquet_jl_tree_identity(candidate, item) + return realpath(candidate) + catch error + push!(failures, "$candidate: $(sprint(showerror, error))") + end + end + detail = isempty(failures) ? "no candidate exists" : join(failures, "; ") + error("exact Parquet.jl dependency source is unavailable for " * + item["name"] * ": " * detail) +end + +function locate_parquet_jl_artifact(item, depots) + relative = joinpath("artifacts", item["git_tree_sha1"]) + failures = String[] + for depot in depots + candidate = joinpath(depot, relative) + ispath(candidate) || continue + try + islink(candidate) && error("artifact root is a symbolic link") + parquet_jl_tree_identity(candidate, item) + return realpath(candidate) + catch error + push!(failures, "$candidate: $(sprint(showerror, error))") + end + end + detail = isempty(failures) ? "no candidate exists" : join(failures, "; ") + error("exact Parquet.jl native artifact is unavailable for " * + item["package"] * ":" * item["name"] * ": " * detail) +end + +function copy_parquet_jl_tree(source::AbstractString, + destination::AbstractString, item) + ispath(destination) && error("private Julia depot target already exists") + mkpath(dirname(destination)) + before = parquet_jl_tree_identity(source, item) + cp(source, destination; follow_symlinks=false) + require_gate(parquet_jl_tree_identity(source, item) == before, + "source tree changed while copied: $source") + require_gate(parquet_jl_tree_identity(destination, item) == before, + "private Julia depot copy differs: $destination") + return +end + +function set_parquet_jl_tree_locked!(root::AbstractString, locked::Bool) + if !locked + root_mode = stat(root).mode & 0o777 + chmod(root, root_mode | 0o700) + for (directory, children, files) in walkdir(root; follow_symlinks=false) + for name in children + path = joinpath(directory, name) + islink(path) && continue + mode = stat(path).mode & 0o777 + chmod(path, mode | 0o700) + end + for name in files + path = joinpath(directory, name) + islink(path) && continue + mode = stat(path).mode & 0o777 + chmod(path, mode | 0o200) + end + end + return + end + directories = String[] + for (directory, children, files) in walkdir(root; follow_symlinks=false) + push!(directories, directory) + for name in children + islink(joinpath(directory, name)) && continue + end + for name in files + path = joinpath(directory, name) + islink(path) && continue + mode = stat(path).mode & 0o777 + chmod(path, mode & 0o555) + end + end + for directory in reverse(directories) + mode = stat(directory).mode & 0o777 + chmod(directory, mode & 0o555) + end + return +end + +function validate_private_parquet_jl_depot(depot::AbstractString, descriptor) + expected_top = sort(["artifacts", "packages"]) + require_gate(sort(readdir(depot)) == expected_top, + "private Julia depot top-level inventory differs") + for dependency in descriptor["dependency"] + root = checked_directory(depot, + joinpath("packages", dependency["name"], dependency["depot_slug"])) + parquet_jl_tree_identity(root, dependency) + end + for artifact in descriptor["artifact"] + root = checked_directory(depot, + joinpath("artifacts", artifact["git_tree_sha1"])) + parquet_jl_tree_identity(root, artifact) + end + for forbidden in ("compiled", "config", "environments", "logs", + "prefs", "registries", "scratchspaces") + require_gate(!ispath(joinpath(depot, forbidden)), + "private Julia depot contains forbidden state: $forbidden") + end + return +end + +function checked_directory(root::AbstractString, relative::AbstractString) + safe_relative(replace(relative, Base.Filesystem.path_separator => '/')) || + error("unsafe relative directory: $relative") + root_path = realpath(root) + candidate = joinpath(root_path, relative) + islink(candidate) && error("pinned directory is a symbolic link: $relative") + isdir(candidate) || error("missing directory: $relative") + resolved = realpath(candidate) + startswith(resolved, root_path * Base.Filesystem.path_separator) || + error("directory escapes its source root: $relative") + return resolved +end + +function with_private_parquet_jl_depot(f, descriptor) + depots = parquet_jl_source_depots() + return mktempdir() do directory + depot = joinpath(directory, "depot") + mkdir(depot) + depot = realpath(depot) + for dependency in descriptor["dependency"] + source = locate_parquet_jl_dependency(dependency, depots) + destination = joinpath(depot, "packages", dependency["name"], + dependency["depot_slug"]) + copy_parquet_jl_tree(source, destination, dependency) + end + for artifact in descriptor["artifact"] + source = locate_parquet_jl_artifact(artifact, depots) + destination = joinpath(depot, "artifacts", + artifact["git_tree_sha1"]) + copy_parquet_jl_tree(source, destination, artifact) + end + validate_private_parquet_jl_depot(depot, descriptor) + set_parquet_jl_tree_locked!(depot, true) + try + result = f(depot) + validate_private_parquet_jl_depot(depot, descriptor) + return result + finally + set_parquet_jl_tree_locked!(depot, false) + end + end +end + +function parquet_jl_runtime_identity(root::AbstractString, descriptor) + inventory = validate_bounded_tree(root; + max_entries=Int(descriptor["julia_runtime_entry_count"]), + max_file_bytes=min(Int(descriptor["julia_runtime_total_bytes"]), + 1024 * 1024 * 1024), + max_total_bytes=Int(descriptor["julia_runtime_total_bytes"])) + require_gate(inventory.entries == descriptor["julia_runtime_entry_count"], + "Julia runtime entry count differs") + require_gate(inventory.bytes == descriptor["julia_runtime_total_bytes"], + "Julia runtime byte count differs") + require_gate(inventory.sha256 == descriptor["julia_runtime_tree_sha256"], + "Julia runtime tree hash differs") + executable = checked_file(root, "bin/julia") + require_gate(file_sha256(executable) == descriptor["julia_executable_sha256"], + "Parquet.jl gate Julia executable differs") + return inventory +end + +function with_private_parquet_jl_runtime_source(f, source::AbstractString, + descriptor) + source = realpath(source) + before = parquet_jl_runtime_identity(source, descriptor) + return mktempdir() do directory + runtime = joinpath(directory, "runtime") + cp(source, runtime; follow_symlinks=false) + runtime = realpath(runtime) + require_gate(parquet_jl_runtime_identity(source, descriptor) == before, + "source Julia runtime changed while copied") + require_gate(parquet_jl_runtime_identity(runtime, descriptor) == before, + "private Julia runtime copy differs") + set_parquet_jl_tree_locked!(runtime, true) + try + result = f(runtime) + require_gate(parquet_jl_runtime_identity(runtime, descriptor) == before, + "private Julia runtime changed while executed") + require_gate(parquet_jl_runtime_identity(source, descriptor) == before, + "source Julia runtime changed while gate executed") + return result + finally + set_parquet_jl_tree_locked!(runtime, false) + end + end +end + +function with_private_parquet_jl_runtime(f, descriptor) + source = realpath(normpath(joinpath(Sys.BINDIR, ".."))) + return with_private_parquet_jl_runtime_source(f, source, descriptor) +end + +function parquet_jl_child_command(gate_root, depot, runtime, descriptor, + descriptor_sha256::String) + executable = checked_file(runtime, "bin/julia") + arguments = String[ + executable, + "--startup-file=no", + "--history-file=no", + "--compiled-modules=no", + "--pkgimages=no", + "--threads=1", + "--project=$gate_root", + gate_file(gate_root, descriptor["bootstrap_file"]), + ] + command = Cmd(Cmd(arguments); dir=gate_root) + return isolated_command(command, + "JULIA_DEPOT_PATH" => depot, + "JULIA_LOAD_PATH" => "@:@stdlib", + "JULIA_NUM_THREADS" => "1", + "JULIA_PKG_OFFLINE" => "true", + "JULIA_PKG_PRECOMPILE_AUTO" => "0", + "JULIA_PKG_SERVER" => "", + "PARQUET_N6_PRODUCER_DESCRIPTOR_SHA256" => descriptor_sha256, + "OPENBLAS_NUM_THREADS" => "1") +end + +function with_verified_parquet_jl_runtime(f, manifest, capabilities, snapshots, + gate_root) + descriptor = validate_parquet_jl_descriptor(manifest, capabilities, snapshots) + require_gate(VERSION == v"1.12.6", + "Parquet.jl evidence gate requires Julia 1.12.6") + require_gate(!ispath(joinpath(gate_root, "LocalPreferences.toml")), + "Parquet.jl gate has LocalPreferences.toml") + with_private_parquet_jl_runtime(descriptor) do runtime + with_private_parquet_jl_depot(descriptor) do depot + descriptor_sha256 = control_snapshot(snapshots, + manifest["parquet_jl_producer_descriptor_file"]).sha256 + run(parquet_jl_child_command(gate_root, depot, runtime, descriptor, + descriptor_sha256)) + return f(runtime) + end + end +end + +function validate_parquet_jl_gate(manifest, capabilities, snapshots, gate_root) + with_verified_parquet_jl_runtime(manifest, capabilities, snapshots, + gate_root) do _ + return + end + return +end + +function test_parquet_jl_identity_helpers() + mktempdir() do root + mkpath(joinpath(root, "src", "nested")) + project = joinpath(root, "Project.toml") + source = joinpath(root, "src", "Parquet.jl") + nested = joinpath(root, "src", "nested", "value.jl") + write(project, "name = \"Parquet\"\n") + write(source, "module Parquet\nend\n") + write(nested, "const VALUE = 1\n") + expected = ["Project.toml", "src/Parquet.jl", "src/nested/value.jl"] + @test parquet_jl_source_paths(root) == expected + snapshots = Dict{String,FileSnapshot}() + entries = Dict{String,Any}[] + for relative in expected + path = joinpath(root, split(relative, '/')...) + payload = read(path) + digest = bytes2hex(SHA.sha256(payload)) + snapshots[relative] = FileSnapshot(path, payload, digest) + push!(entries, Dict{String,Any}( + "path" => relative, "sha256" => digest)) + end + baseline = parquet_jl_source_composite(entries, snapshots) + @test occursin(SHA256_PATTERN, baseline) + changed = copy(snapshots) + payload = UInt8[codeunits("module Parquet\nconst CHANGED = true\nend\n")...] + changed["src/Parquet.jl"] = FileSnapshot(source, payload, + bytes2hex(SHA.sha256(payload))) + @test_throws ErrorException parquet_jl_source_composite(entries, changed) + extra = joinpath(root, "src", "extra.jl") + write(extra, "const EXTRA = true\n") + @test parquet_jl_source_paths(root) != expected + rm(extra) + rm(nested) + @test parquet_jl_source_paths(root) != expected + write(nested, "const VALUE = 1\n") + rm(nested) + symlink(source, nested) + @test_throws ErrorException parquet_jl_source_paths(root) + return + end + mktempdir() do root + file = joinpath(root, "value") + write(file, "exact\n") + inventory = validate_bounded_tree(root; max_entries=1, + max_file_bytes=6, max_total_bytes=6) + item = Dict{String,Any}( + "entry_count" => Int64(1), + "total_bytes" => Int64(6), + "tree_sha256" => inventory.sha256, + "git_tree_sha1" => bytes2hex(Pkg.GitTools.tree_hash(root)), + ) + @test parquet_jl_tree_identity(root, item) == inventory + write(file, "other\n") + @test_throws ErrorException parquet_jl_tree_identity(root, item) + write(file, "exact\n") + write(joinpath(root, "extra"), "x") + @test_throws ErrorException parquet_jl_tree_identity(root, item) + return + end + mktempdir() do root + mkpath(joinpath(root, "bin")) + mkpath(joinpath(root, "lib")) + executable = joinpath(root, "bin", "julia") + write(executable, "runtime\n") + chmod(executable, 0o755) + write(joinpath(root, "lib", "payload"), "library\n") + inventory = validate_bounded_tree(root; max_entries=4, + max_file_bytes=16, max_total_bytes=16) + descriptor = Dict{String,Any}( + "julia_runtime_entry_count" => inventory.entries, + "julia_runtime_total_bytes" => inventory.bytes, + "julia_runtime_tree_sha256" => inventory.sha256, + "julia_executable_sha256" => file_sha256(executable), + ) + @test parquet_jl_runtime_identity(root, descriptor) == inventory + with_private_parquet_jl_runtime_source(root, descriptor) do runtime + @test runtime != realpath(root) + @test parquet_jl_runtime_identity(runtime, descriptor) == inventory + @test stat(runtime).mode & 0o222 == 0 + @test stat(joinpath(runtime, "bin", "julia")).mode & 0o111 != 0 + return + end + wrong = copy(descriptor) + wrong["julia_runtime_entry_count"] += 1 + @test_throws ErrorException parquet_jl_runtime_identity(root, wrong) + return + end + return +end diff --git a/test/conformance/n6/julia/runtests.jl b/test/conformance/n6/julia/runtests.jl new file mode 100644 index 0000000..ffc4988 --- /dev/null +++ b/test/conformance/n6/julia/runtests.jl @@ -0,0 +1,215 @@ +using Parquet +using Test + +include(joinpath(@__DIR__, "N6ParquetJLHarness.jl")) + +const N6H = N6ParquetJLHarness +const N6MD = Parquet.Metadata +const N6Model = N6H.Model + +function _n6unknownorder() + raw = Parquet.Thrift.RawField(77, Parquet.Thrift.STRUCT, UInt8[0x00]) + return N6MD.ColumnOrder(unknown_fields=(raw,)) +end + +function _n6factpair(family::Symbol, createdby::String; + order::Symbol=:type, total::Int64=Int64(4), nulls=nothing, + lower::Vector{UInt8}=UInt8[0x01], + upper::Vector{UInt8}=UInt8[0x02], limit::Int64=Int64(4096)) + family in (:modern, :deprecated) || throw(ArgumentError( + "unsupported statistics family $family")) + element = N6MD.SchemaElement(type_=N6MD.Type.BYTE_ARRAY, + repetition_type=N6MD.FieldRepetitionType.OPTIONAL, name="value") + root = N6MD.SchemaElement( + repetition_type=N6MD.FieldRepetitionType.REQUIRED, name="schema", + num_children=Int32(1)) + schema = Parquet.Schema(N6MD.SchemaElement[root, element]) + statistics = family === :modern ? N6MD.Statistics( + min_value=lower, max_value=upper, null_count=nulls) : + N6MD.Statistics(min=lower, max=upper, null_count=nulls) + metadata = N6MD.ColumnMetaData(type_=N6MD.Type.BYTE_ARRAY, + encodings=[N6MD.Encoding.PLAIN], path_in_schema=["value"], + codec=N6MD.CompressionCodec.UNCOMPRESSED, num_values=total, + total_uncompressed_size=Int64(0), total_compressed_size=Int64(0), + data_page_offset=Int64(0), statistics=statistics) + productionorders = order === :missing ? nothing : N6MD.ColumnOrder[ + order === :type ? N6MD.ColumnOrder(TYPE_ORDER=N6MD.TypeDefinedOrder()) : + order === :unknown ? _n6unknownorder() : throw(ArgumentError( + "unsupported declared order $order"))] + modelorders = order === :missing ? nothing : N6Model.DeclaredOrder[ + order === :type ? N6Model.ORDER_TYPE : N6Model.ORDER_FUTURE] + rawstatistics = family === :modern ? N6Model.RawStatistics( + modern_lower=lower, modern_upper=upper, null_count=nulls) : + N6Model.RawStatistics(deprecated_lower=lower, + deprecated_upper=upper, null_count=nulls) + spec = N6Model.LeafSpec(N6Model.PHYSICAL_BYTE_ARRAY) + modeled = N6Model.interpret_statistics(spec, total, rawstatistics, + modelorders; created_by=createdby, + limits=N6Model.ModelLimits(max_statistics_value_bytes=limit)) + production = Parquet._statisticsfacts(schema, 1, createdby, + productionorders, metadata; + limits=Parquet.Limits(max_statistics_value_bytes=limit)) + selectedorder = modelorders === nothing ? nothing : only(modelorders) + N6H.comparefacts(modeled, production, spec, selectedorder) + return modeled, production +end + +@testset "N6 Parquet.jl deterministic fixture and evidence harness" begin + first_output = N6H.buildharness() + second_output = N6H.buildharness() + @test length(first_output.files) == 12 + @test length(first_output.checked) == 12 + @test first_output.files == second_output.files + @test first_output.evidence == second_output.evidence + @test count(==(UInt8('\n')), first_output.evidence) == 96 + @test sum(length(case.column_records) for case in first_output.checked) == 52 + @test sum(1 + length(case.column_records) for case in first_output.checked) == 64 + @test length(unique(Base.first(pair) for pair in first_output.files)) == 12 + @test all(pair -> length(last(pair)) <= N6H.MAX_GENERATED_BYTES, + first_output.files) + grouped = [case for case in first_output.checked if + case.declaration["comparison_group"] == + "julia-reader-no-pruning-v1"] + @test length(grouped) == 5 + @test length(Set(case.no_pruning_sha256 for case in grouped)) == 1 + @test length(Set(case.assertion_facts["body_sha256"] for case in grouped)) == 1 + @test length(Set(case.logical_values_sha256 for case in grouped)) == 1 + @test length(Set(case.assertion_facts["range_trace_sha256"] + for case in grouped)) == 1 + @test length(Set(case.assertion_facts["read_count"] + for case in grouped)) == 1 +end + +@testset "N6 harness destination and independence guards" begin + @test_throws ArgumentError N6H._safeoutput("../escape.parquet") + @test_throws ArgumentError N6H._safeoutput("generated/nested/escape.parquet") + @test_throws ArgumentError N6H._safeoutput("generated/escape\\file.parquet") + # Durable publication fsyncs its destination directory and relies on POSIX + # rename and symlink behavior, so the harness refuses to publish anywhere else. + # Exercise those guarantees only where they exist. + Sys.isunix() && mktempdir() do directory + path = joinpath(directory, "value.bin") + N6H._atomicreplacebytes(path, UInt8[0x01, 0x02]) + @test read(path) == UInt8[0x01, 0x02] + @test filemode(path) & 0o777 == 0o644 + N6H._atomicreplacebytes(path, UInt8[0x03]) + @test read(path) == UInt8[0x03] + link = joinpath(directory, "link.bin") + symlink(path, link) + @test_throws ArgumentError N6H._atomicreplacebytes(link, UInt8[0x04]) + @test read(path) == UInt8[0x03] + directory_target = joinpath(directory, "directory.bin") + mkdir(directory_target) + @test_throws ArgumentError N6H._atomicreplacebytes(directory_target, + UInt8[0x04]) + first = joinpath(directory, "first.bin") + write(first, UInt8[0x05]) + @test_throws ArgumentError N6H._atomicreplacebatch( + Pair{String,Vector{UInt8}}[ + first => UInt8[0x06], + directory_target => UInt8[0x07], + ]) + @test read(first) == UInt8[0x05] + syncs = Ref(0) + failfirstsync = function(path) + syncs[] += 1 + syncs[] == 1 && error("injected directory sync failure") + return N6H._fsyncdirectory(path) + end + @test_throws ErrorException N6H._atomicreplacebatch( + Pair{String,Vector{UInt8}}[first => UInt8[0x08]]; + syncdirectory=failfirstsync) + @test read(first) == UInt8[0x05] + displacement = joinpath(directory, "displacement.bin") + write(displacement, UInt8[0x09]) + reservationobserved = Ref(false) + replacement = N6H._renameoverreserved(displacement, directory; + renamefile=function(source, target) + reservationobserved[] = isfile(target) + return N6H._renamefile(source, target) + end) + @test reservationobserved[] + @test !ispath(displacement) + @test read(replacement) == UInt8[0x09] + rm(replacement) + @test N6H._stablefilebytes(first, Int64(1), "test input") == UInt8[0x05] + @test_throws ArgumentError N6H._stablefilebytes(first, Int64(0), + "test input") + inputlink = joinpath(directory, "input-link.bin") + symlink(first, inputlink) + @test_throws ArgumentError N6H._stablefilebytes(inputlink, Int64(1), + "test input") + end + stale, stream = mktemp(joinpath(N6H.N6_ROOT, "generated")) + try + write(stream, zeros(UInt8, 1024)) + close(stream) + @test_throws ArgumentError N6H._checkbytes(stale, UInt8[0x01]) + finally + isopen(stream) && close(stream) + ispath(stale) && rm(stale; force=true) + end + modelsource = String(copy(N6H.FROZEN_MODEL_BYTES)) + @test !occursin(r"(?m)^\s*(using|import)\s+Parquet\b", modelsource) + for (directory, _, names) in walkdir(joinpath(N6H.REPO_ROOT, "src")) + for name in names + endswith(name, ".jl") || continue + @test !occursin("N6StatisticsModel", + read(joinpath(directory, name), String)) + end + end + if VERSION == N6H.CANONICAL_WRITER_VERSION + @test isnothing(N6H._checkwriteridentity()) + else + @test_throws ArgumentError N6H._checkwriteridentity() + end +end + +@testset "N6 statistics policy cross-precedence" begin + producers = ("parquet-cpp version 1.2.9", + "parquet-mr version 1.9.9") + for createdby in producers + modeled, production = _n6factpair(:modern, createdby) + @test modeled.comparator == N6Model.COMPARATOR_UNSIGNED_BYTES + @test modeled.trust.state == N6Model.TRUST_UNTRUSTED + @test production.comparison == :unsigned_bytes + modeled, production = _n6factpair(:deprecated, createdby) + @test modeled.comparator == N6Model.COMPARATOR_UNDEFINED + @test production.comparison == :undefined + @test modeled.lower.reason == :deprecated_order_mismatch + @test production.lower.reason == :deprecated_order_mismatch + for order in (:missing, :unknown) + modeled, production = _n6factpair(:modern, createdby; + order=order, total=Int64(4), nulls=Int64(4)) + expected = order === :missing ? :missing_column_orders : + :unknown_column_order + @test modeled.occupancy == N6Model.OCCUPANCY_EMPTY + @test modeled.trust.state == N6Model.TRUST_UNTRUSTED + @test modeled.lower.reason == expected + @test production.occupancy == :no_non_null + end + for equal in (false, true) + lower = fill(UInt8(0x61), 5) + upper = equal ? copy(lower) : fill(UInt8(0x62), 5) + modeled, production = _n6factpair(:modern, createdby; + lower=lower, upper=upper, limit=Int64(4)) + @test modeled.trust.state == N6Model.TRUST_UNTRUSTED + @test modeled.lower.reason == :over_limit + @test modeled.upper.reason == :over_limit + @test production.lower.reason == :over_limit + @test production.upper.reason == :over_limit + end + end +end + +@testset "N6 generated output freshness" begin + output = N6H.runharness(:check) + @test length(output.files) == 12 + @test isfile(N6H.EVIDENCE_FILE) + @test !islink(N6H.EVIDENCE_FILE) + for (relative, _) in output.files + path = N6H._safeoutput(relative) + @test isfile(path) + @test !islink(path) + end +end diff --git a/test/conformance/n6/manifest.toml b/test/conformance/n6/manifest.toml new file mode 100644 index 0000000..ae681d8 --- /dev/null +++ b/test/conformance/n6/manifest.toml @@ -0,0 +1,434 @@ +manifest_version = 1 +gate = "n6-a-preproduction" +status = "preproduction" +plan_file = "docs/dev/n6-statistics-plan.md" +plan_sha256 = "15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304" +capabilities_file = "test/conformance/n6/capabilities.toml" +capabilities_sha256 = "50f1db2361e63fca0be49790da5bce7104ece3fa606551e713352bec7b07a419" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +fixture_manifest_sha256 = "670b2b1cbc0755eaa61c4638d4dc78a5ec12808c80cff56368ba40ba483e9c25" +corpus_manifest_file = "test/conformance/n6/corpus-files.sha256" +corpus_manifest_sha256 = "10c5e8fc52bd1d675401fd417c790e45d8103a84e636e42adec20376371c1991" +evidence_schema_file = "test/conformance/n6/evidence.schema.json" +evidence_schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +artifact_manifest_file = "test/conformance/n6/artifacts.sha256" +artifact_manifest_sha256 = "f681a5402a3ff9905248799a41a85bd81fa86b11b6e89b4d57c6dc8a81850d0b" +model_producer_descriptor_file = "test/conformance/n6/normalizers/model-producer.toml" +model_producer_descriptor_sha256 = "876e7382758b94c895e57769e8fb52216ac626394ff4026dee00f92812db7f3c" +parquet_jl_producer_descriptor_file = "test/conformance/n6/julia/parquet-jl-producer.toml" +parquet_jl_producer_descriptor_sha256 = "4cf7759e22159ab3c6133bcecc3ba94b5857485581e586bf8dad49bc325a3a5e" +parquet_jl_source_composite_sha256 = "ea75000a8b4505c73efe50476a45dfe427ed5c8f7123fbaf244390f2b26b0c80" +supported_platforms = ["macos-15-arm64"] +publication_authorized = false +oracle_lock_authorized = false +planned_evidence = [] + +[evidence_limits] +max_inputs = 10 +max_file_bytes = 33554432 +max_total_bytes = 67108864 +max_line_bytes = 1048576 +max_records_per_input = 328 +max_records_total = 3280 + +[[source]] +id = "parquet-format" +url = "https://github.com/apache/parquet-format.git" +version = "2.13.0" +tag = "apache-parquet-format-2.13.0" +tag_revision = "a9f9c3a52bd1d6309038f4d2d3a308978b55c377" +revision = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" +status = "verified" +root_env = "PARQUET_N6_FORMAT_ROOT" +files = [ + { file = "src/main/thrift/parquet.thrift", sha256 = "53bb8fc9b96469d7ca694121ead839e449e5156d7bf79f0df728cdd72796df38" }, + { file = "LogicalTypes.md", sha256 = "4e5748116514c8682fdf1920050cf13ed900bd4bfc9ac9995a0a92841440eeac" }, +] + +[[source]] +id = "parquet-testing" +url = "https://github.com/apache/parquet-testing.git" +version = "snapshot" +tag = "" +tag_revision = "" +revision = "09f3cdbde45302f0f0c689c950e465e98a9df960" +status = "verified" +root_env = "PARQUET_N6_TESTING_ROOT" +files = [] + +[[source]] +id = "parquet-java" +url = "https://github.com/apache/parquet-java.git" +version = "1.17.1" +tag = "apache-parquet-1.17.1" +tag_revision = "1f54ba44afb285fecbaf54bde5c0afa259327fc4" +revision = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +status = "verified" +root_env = "PARQUET_N6_JAVA_ROOT" +files = [ + { file = "parquet-common/src/main/java/org/apache/parquet/VersionParser.java", sha256 = "a8c54497632bbfacd2da1d52fcf83e3ee6c7e4c0134048c4649bd50055866c87" }, + { file = "parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java", sha256 = "1344e06d6644c9b8d027d03d3e8a595fc60f029eef5e6156b63248a4ee346dc7" }, + { file = "parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java", sha256 = "258cb4a4b5f6b6a846c51ebf13d868fe4907ee3c8dfa6e5e73484e927564d1cd" }, +] + +[[source]] +id = "arrow-rs" +url = "https://github.com/apache/arrow-rs.git" +version = "59.2.0" +tag = "59.2.0" +tag_revision = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +revision = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +status = "verified" +root_env = "PARQUET_N6_ARROW_RS_ROOT" +files = [ + { file = "parquet/src/parquet_thrift.rs", sha256 = "efdde5da47d94685096ae9c197bc657d15269cdd53f53ea212aff2cd4122075c" }, + { file = "parquet/src/file/statistics.rs", sha256 = "9e22a3c8e10cc3058b41b3262dfbfa2213899e09f52faec13991429957dd2c7b" }, + { file = "parquet/src/file/metadata/mod.rs", sha256 = "34f64fe9fa851edf14745d4ebdb126d00c29ad4dccb10dec90559e6bf943b84d" }, +] + +[[source]] +id = "apache-arrow-cpp-policy" +url = "https://github.com/apache/arrow.git" +version = "source" +tag = "" +tag_revision = "" +revision = "515410b2a14ac766258e00b07eab9e5ee2692a62" +status = "verified" +root_env = "PARQUET_N6_ARROW_CPP_ROOT" +files = [ + { file = "cpp/src/parquet/metadata.cc", sha256 = "126a081d2a8270a99b2f456a25a40113567bc5f2e03a2feb266354cfa440c5f1" }, +] + +[[source]] +id = "pyarrow" +url = "https://github.com/apache/arrow.git" +version = "25.0.1" +tag = "apache-arrow-25.0.1" +tag_revision = "8bf34803daea7c13f806bf29ee7b09d16773acb2" +revision = "beccec0d0c451b7aa3e4530416ac431b3c035c69" +status = "planned" +root_env = "PARQUET_N6_PYARROW_ROOT" +files = [] + +[[source]] +id = "duckdb" +url = "https://github.com/duckdb/duckdb.git" +version = "1.5.5" +tag = "v1.5.5" +tag_revision = "d8cdaa33fda8df955cc76ef58a280f68f4cd43fa" +revision = "d8cdaa33fda8df955cc76ef58a280f68f4cd43fa" +status = "planned" +root_env = "PARQUET_N6_DUCKDB_ROOT" +files = [] + +[[source]] +id = "cpython-3.12" +url = "https://github.com/python/cpython.git" +version = "3.12.8" +tag = "v3.12.8" +tag_revision = "3a523043165785601895d7cdc74713dfa64d630d" +revision = "2dc476bcb9142cd25d7e1d52392b73a3dcdf1756" +status = "verified" +root_env = "PARQUET_N6_CPYTHON_312_ROOT" +files = [] + +[[source]] +id = "cpython-3.14" +url = "https://github.com/python/cpython.git" +version = "3.14.2" +tag = "v3.14.2" +tag_revision = "a1d0069daf8e85b25a0c3f96abc43182be6d429e" +revision = "df793163d5821791d4e7caf88885a2c11a107986" +status = "verified" +root_env = "PARQUET_N6_CPYTHON_314_ROOT" +files = [] + +[[toolchain]] +id = "julia-1.10" +status = "verified" +version = "1.10.11" +platform = "aarch64-apple-darwin" +scope = "Independent model and package compatibility." +artifacts = [ + { name = "julia-1.10.11-executable", sha256 = "6f687953e48958fc6596962379691d1c8a1720d3a9ff39c1e3113888e43bd8ae" }, + { name = "julia-1.10.11-runtime-tree-sha256-v1", sha256 = "c784c03af8ab52e48aa6f57ce8aa06a3c9160671054301ab6c3dfebf2e582525" }, +] + +[[toolchain]] +id = "julia-1.12" +status = "verified" +version = "1.12.6" +platform = "aarch64-apple-darwin" +scope = "Independent model and package compatibility." +artifacts = [ + { name = "julia-1.12.6-executable", sha256 = "9ad38bea81ecace044a4bdef2a0246dee94cb8a44c9420809cc00f9872651c64" }, + { name = "julia-1.12.6-runtime-tree-sha256-v1", sha256 = "273ec71de498a36c77a7e4bb3af4a3f75c338bd1cfe255cab30805b6a2cda76e" }, +] + +[[toolchain]] +id = "parquet-jl-evidence" +status = "verified" +version = "parquet-jl-1.0.0-DEV_julia-1.12.6" +platform = "macos-15-arm64" +scope = "Exact Parquet.jl source, manifest, private dependency depot, native artifacts, bootstrap, and N6 harness." +artifacts = [ + { name = "parquet-jl-producer.toml", sha256 = "4cf7759e22159ab3c6133bcecc3ba94b5857485581e586bf8dad49bc325a3a5e" }, +] + +[[toolchain]] +id = "raw-java" +status = "verified" +version = "temurin-21.0.8+9_thrift-0.23.0" +platform = "macos-15-arm64" +scope = "Separate raw Parquet 2.13 Compact-Thrift scanner." +artifacts = [ + { name = "toolchain.env", sha256 = "8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629" }, + { name = "temurin-jdk-archive", sha256 = "59422c2292ae4e76b87e00d8808dbe49cffa39af731e08bb0292ddb0af4e0261" }, + { name = "temurin-java", sha256 = "0045ae168ee132bbf469a26fb17dac6d1dee431c9b7826474f3b6ee574a997c9" }, + { name = "temurin-javac", sha256 = "7be7937fc6bae0ca89f0866f9ce94fc40a935dfb87806d3c701eca3402cfb90a" }, + { name = "thrift-compiler", sha256 = "5ee94e75371f7d0b2467db3acdb67b8b3814fcae3748c0ef078a490a15c57e11" }, + { name = "libthrift-0.23.0.jar", sha256 = "8b41b67a5ff13c371ab18b6d34506121dcecf11372829f7d50115cfb1bf72d42" }, + { name = "libthrift-0.23.0.pom", sha256 = "eeadf7b9d1e22ac01985fe552384ceecf44018cae93e7a23d6b0466f856330f3" }, + { name = "slf4j-api-1.7.36.jar", sha256 = "d3ef575e3e4979678dc01bf1dcce51021493b4d11fb7f1be8ad982877c16a1c0" }, + { name = "slf4j-nop-1.7.36.jar", sha256 = "c214958b07816cb4412b30c7bdbd4308ffdc6ba2a83767b8f3a9229cbd9274d6" }, + { name = "generated-source-manifest", sha256 = "f738c7346ad1dd70faafd54815f829b8587a2b0397ff6b6e6710a3a7276cac09" }, + { name = "parquet-2.13-idl", sha256 = "53bb8fc9b96469d7ca694121ead839e449e5156d7bf79f0df728cdd72796df38" }, +] + +[[toolchain]] +id = "rust-arrow-rs" +status = "verified" +version = "rustc-and-cargo-1.96.1" +platform = "linux-amd64-offline-harness" +scope = "Pinned N5 Arrow Rust oracle image and verified N6 host harness." +artifacts = [ + { name = "rust-channel-manifest", sha256 = "87eb76c53073e72b766083bed5530820694253b832a762d8385bda5759f03975" }, + { name = "rust-distribution", sha256 = "d29ccb1559a177c4e72291f6e5f629de7fe8885e7521ca47802627544b121e95" }, + { name = "n5-arrow-rs-Cargo.lock", sha256 = "5797b71e20f0a4eda601c8c126336e4daffa2dfc1d9a992d6762cd3fc0b902cc" }, + { name = "n5-arrow-rs-UPSTREAM.toml", sha256 = "d547dbed70c01b77f076a6f020767693751d869c3ff5f5279c5d6ce06f152218" }, + { name = "n5-oracle-Dockerfile", sha256 = "68731325b41332ff580ffe0fa007d6d9e0ec09b5f3ffe09511b99ec7096280db" }, + { name = "oracles/arrow-rs/toolchain.toml", sha256 = "c141bde2eea6442bcd8b3c7ad9f8aeadb04e7ad1080f5dc7b7cba0a2dae4cfe5" }, + { name = "parquet-jl-n6-arrow-rs-metadata", sha256 = "a26a0a29f99adde346800e85ee66544226d2105a040b40eb3536e8d629d8c301" }, +] + +[[toolchain]] +id = "parquet-java-interop" +status = "verified" +version = "parquet-java-1.17.1_temurin-21.0.8+9_cpython-3.12.8" +platform = "macos-15-arm64" +scope = "Pinned Parquet Java N6 value, statistics, and producer-policy harness." +artifacts = [ + { name = "oracles/parquet-java/toolchain.toml", sha256 = "a4459b555778b1979de10510fd5291372ffb9f41166539868ce3639494d5a452" }, + { name = "cpython-distribution-archive", sha256 = "dfb8a4c87116538717105ef3dec3668ae07590a5b5532109fec3ccad90be2fbc" }, + { name = "cpython-executable", sha256 = "d6b64f766d3b08326aa10cdb37c9d922e3af38b57ec07caa28b894e5fccf6e69" }, + { name = "cpython-clean-tree-sha256-v1", sha256 = "e3b7dcdffba67f605b0fa3318656387e8d4265d34531c2bdbc43d3aba4a033ec" }, + { name = "temurin-jdk-archive", sha256 = "59422c2292ae4e76b87e00d8808dbe49cffa39af731e08bb0292ddb0af4e0261" }, + { name = "temurin-java", sha256 = "0045ae168ee132bbf469a26fb17dac6d1dee431c9b7826474f3b6ee574a997c9" }, + { name = "temurin-javac", sha256 = "7be7937fc6bae0ca89f0866f9ce94fc40a935dfb87806d3c701eca3402cfb90a" }, + { name = "temurin-release", sha256 = "8e98b265f9a6fd3db04d2535108497897e87f1b3821270cf10ffa937463dc2ee" }, + { name = "temurin-tree-sha256-v1", sha256 = "7ffead12e2614843ffcc77e36d1bd142b921d91381d4ef12827c32c54369d209" }, + { name = "parquet-cli-1.17.1-runtime.jar", sha256 = "d0173051493c506a298c691e555a41a682a405895fd0c8cc429a7e1cb1fcc711" }, + { name = "hadoop-client-api-3.3.0.jar", sha256 = "d549ba6d131fd6c8e5d42a78dab5c790950edd6258523dedc556b537ca6654aa" }, + { name = "hadoop-client-runtime-3.3.0.jar", sha256 = "2ba23f1e1dbb03e73600a41fcb187ad2626529684ed226085a91b0a0d6d67ee5" }, + { name = "parquet-java-n6-harness.jar", sha256 = "7d6de1067e4e01de65f5868c8643f4a68fe4ff6afc50759716d685bd7b9e764c" }, +] + +[[toolchain]] +id = "python-interop" +status = "verified" +version = "cpython-3.12.8+20250115" +platform = "macos-15-arm64" +scope = "Source-attested clean CPython, PyArrow, and DuckDB N6 harness." +artifacts = [ + { name = "cpython-distribution-archive", sha256 = "dfb8a4c87116538717105ef3dec3668ae07590a5b5532109fec3ccad90be2fbc" }, + { name = "cpython-executable", sha256 = "d6b64f766d3b08326aa10cdb37c9d922e3af38b57ec07caa28b894e5fccf6e69" }, + { name = "cpython-clean-tree-sha256-v1", sha256 = "e3b7dcdffba67f605b0fa3318656387e8d4265d34531c2bdbc43d3aba4a033ec" }, + { name = "pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", sha256 = "df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9" }, + { name = "duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", sha256 = "f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae" }, + { name = "toolchains/pyarrow.toml", sha256 = "0e1e7fa951f82f12ddfdbeb3865936763c1818f1f20a655d2646b1186cbcd958" }, + { name = "toolchains/duckdb.toml", sha256 = "a3c20486eae1ec54266ca94e9a29f8fa0c7e46bdddbb829d28bc92991ddda6ae" }, +] + +[[toolchain]] +id = "jsonschema-validator" +status = "verified" +version = "cpython-3.14.2+20260127_jsonschema-4.26.0" +platform = "macos-15-arm64" +scope = "Draft 2020-12 schema meta-validation and semantic evidence checks." +distribution_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260127/cpython-3.14.2%2B20260127-aarch64-apple-darwin-install_only_stripped.tar.gz" +tree_policy = "extract-strip-site-packages-bytecode-v1" +artifacts = [ + { name = "cpython-distribution-archive", sha256 = "2f82ed0cf902c8a2863a3fc5864909645c761e4e0a96525850c9ef8f7e3b79e7" }, + { name = "cpython-executable", sha256 = "3d6400b63b150164e89a690d9813af8b0eb420af9f336ef1f6c5102c6da60eae" }, + { name = "cpython-clean-tree-sha256-v1", sha256 = "0858c90317b52f4224a5daf095a76feaae771869700876184b4ecd16f544ea4a" }, + { name = "jsonschema-4.26.0-py3-none-any.whl", sha256 = "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce" }, + { name = "attrs-25.4.0-py3-none-any.whl", sha256 = "adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373" }, + { name = "jsonschema_specifications-2025.9.1-py3-none-any.whl", sha256 = "98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe" }, + { name = "referencing-0.37.0-py3-none-any.whl", sha256 = "381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231" }, + { name = "rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", sha256 = "ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be" }, +] + +[[frozen_model]] +file = "test/conformance/n6/model/N6StatisticsModel.jl" +sha256 = "32c090ed6e0c6af49eabf3f96afc6e17dff630c4e89372367e693202e87c4262" + +[[frozen_model]] +file = "test/conformance/n6/model/runtests.jl" +sha256 = "c74aab7a4d522a77443bba61070bae51c15bfec75a917bcd5ea16331d2307691" + +[[frozen_model]] +file = "test/conformance/n6/model/cases.toml" +sha256 = "925da4abeb448033b291e8f2a1ed8d68f6af30e5a4994d1e576f757581c5ad7b" + +[[frozen_model]] +file = "test/conformance/n6/model/README.md" +sha256 = "143290750312ae294c9954daf3d30d9d79b8627c67059ee001ba2228fa803814" + +[[frozen_evidence]] +id = "raw-java-apache-corpus" +status = "verified" +authority = "n6-raw-java" +toolchain_sha256 = "8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629" +file = "test/conformance/n6/evidence/raw-java-apache-corpus.raw.jsonl" +format = "raw-jsonl" +storage = "gate-generated" +schema_file = "test/conformance/n6/oracles/raw-java/evidence.schema.json" +schema_sha256 = "8fb8144d6838906141f827736785540ac9ceaf638d60399c8d4c1940e34d8f28" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 20 +record_count = 20 +sha256 = "b2ef6982235e13ff44fa8ce4562e2f4e7fd1c20eeaf61968563351ae117df439" +scope = "Gate-generated ephemeral raw Parquet 2.13 footer records for the 20 exact Apache fixtures; the declared repository path is an identity label, not a checked-in materialization, and there is no semantic pass claim." + +[[frozen_evidence]] +id = "normalized-raw-java-apache-corpus" +status = "verified" +authority = "n6-raw-java" +toolchain_sha256 = "8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629" +file = "test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 20 +record_count = 150 +sha256 = "1692d30284b57581993d524d41baa16b43e94b26dd3832bd6d63d689a1bb4ff8" +scope = "Normalized raw wire facts for the exact Apache corpus; no semantic model result." + +[[frozen_evidence]] +id = "normalized-independent-model" +status = "verified" +authority = "n6-independent-model" +toolchain_sha256 = "273ec71de498a36c77a7e4bb3af4a3f75c338bd1cfe255cab30805b6a2cda76e" +file = "test/conformance/n6/evidence/independent-model.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 20 +record_count = 128 +sha256 = "1ed64a95e2fee59c431e35720f20861492822a523b9f3c9f58c2dbc5ea9ca8ff" +upstream_evidence = ["normalized-raw-java-apache-corpus"] +scope = "Independent semantic results combined with reviewed normalized raw inputs." + +[[frozen_evidence]] +id = "normalized-raw-java-generated" +status = "verified" +authority = "n6-raw-java" +toolchain_sha256 = "8608303c0624c5fb9692dc007fb4930c72af744a8a7355a19e8c607b3403f629" +file = "test/conformance/n6/evidence/raw-java-generated.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 12 +record_count = 82 +sha256 = "724c1c2066ac640f35e2c7b7d8fd806b11cff5158417e7bd609b03faf49ca433" +scope = "Normalized raw wire facts for the exact 12-file Parquet.jl generated corpus; no semantic model result." + +[[frozen_evidence]] +id = "normalized-parquet-jl" +status = "verified" +authority = "parquet-jl" +toolchain_sha256 = "4cf7759e22159ab3c6133bcecc3ba94b5857485581e586bf8dad49bc325a3a5e" +file = "test/conformance/n6/evidence/parquet-jl.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 12 +record_count = 96 +sha256 = "61ac2835f42591fe3beb187b84232eaac11d3373c1e89194ff5cb940e810a963" +scope = "Exact generated Julia writer and reader evidence." + +[[frozen_evidence]] +id = "normalized-parquet-java" +status = "verified" +authority = "parquet-java" +toolchain_sha256 = "a4459b555778b1979de10510fd5291372ffb9f41166539868ce3639494d5a452" +file = "test/conformance/n6/evidence/parquet-java.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 19 +record_count = 49 +sha256 = "ea92c9f9aaaf2f5168a02823b11e258cc674ba44be3f777c9b469e126c94a2e5" +upstream_evidence = ["normalized-raw-java-apache-corpus"] +scope = "Exact N6 parquet-java interoperability harness evidence." + +[[frozen_evidence]] +id = "normalized-arrow-rs" +status = "verified" +authority = "arrow-rs" +toolchain_sha256 = "c141bde2eea6442bcd8b3c7ad9f8aeadb04e7ad1080f5dc7b7cba0a2dae4cfe5" +file = "test/conformance/n6/evidence/arrow-rs.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 19 +record_count = 45 +sha256 = "d81b0f65cd11d173425f8339e0a97eda0d805eb26b0757af7766ad2d16254811" +upstream_evidence = ["normalized-raw-java-apache-corpus"] +scope = "Exact N6 Arrow Rust interoperability harness evidence." + +[[frozen_evidence]] +id = "normalized-pyarrow" +status = "verified" +authority = "pyarrow" +toolchain_sha256 = "0e1e7fa951f82f12ddfdbeb3865936763c1818f1f20a655d2646b1186cbcd958" +file = "test/conformance/n6/evidence/pyarrow.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 16 +record_count = 38 +sha256 = "305f12ff6f144d7c7e7d178de43485e3aa49c5d4c4122e1354f7b46581a45811" +upstream_evidence = ["normalized-raw-java-apache-corpus"] +scope = "Exact PyArrow logical-value and exposed statistics evidence." + +[[frozen_evidence]] +id = "normalized-duckdb" +status = "verified" +authority = "duckdb" +toolchain_sha256 = "a3c20486eae1ec54266ca94e9a29f8fa0c7e46bdddbb829d28bc92991ddda6ae" +file = "test/conformance/n6/evidence/duckdb.normalized.jsonl" +format = "normalized-jsonl" +storage = "checked-in" +schema_file = "test/conformance/n6/evidence.schema.json" +schema_sha256 = "5e2b4968d854a3da166b381a48efbd92bbc77d54b91d442736d3154736431e31" +fixture_manifest_file = "test/conformance/n6/fixtures.toml" +case_count = 15 +record_count = 35 +sha256 = "70812c4b76586f479e5c632222b9912fd02cbbae1c52fa5e6a521740d4beb1bb" +upstream_evidence = ["normalized-raw-java-apache-corpus"] +scope = "Exact DuckDB readability and metadata-view evidence." diff --git a/test/conformance/n6/model/N6StatisticsModel.jl b/test/conformance/n6/model/N6StatisticsModel.jl new file mode 100644 index 0000000..bbbc621 --- /dev/null +++ b/test/conformance/n6/model/N6StatisticsModel.jl @@ -0,0 +1,1334 @@ +module N6StatisticsModel + +@enum PhysicalType begin + PHYSICAL_BOOLEAN + PHYSICAL_INT32 + PHYSICAL_INT64 + PHYSICAL_INT96 + PHYSICAL_FLOAT + PHYSICAL_DOUBLE + PHYSICAL_BYTE_ARRAY + PHYSICAL_FIXED_LEN_BYTE_ARRAY +end + +@enum LogicalType begin + LOGICAL_NONE + LOGICAL_STRING + LOGICAL_ENUM + LOGICAL_JSON + LOGICAL_BSON + LOGICAL_UUID + LOGICAL_DECIMAL + LOGICAL_SIGNED_INTEGER + LOGICAL_UNSIGNED_INTEGER + LOGICAL_DATE + LOGICAL_TIME + LOGICAL_TIMESTAMP + LOGICAL_FLOAT16 + LOGICAL_INTERVAL + LOGICAL_UNKNOWN + LOGICAL_VARIANT + LOGICAL_GEOMETRY + LOGICAL_GEOGRAPHY + LOGICAL_LIST + LOGICAL_MAP +end + +@enum TimeUnit begin + TIME_MILLIS + TIME_MICROS + TIME_NANOS +end + +@enum ComparatorKind begin + COMPARATOR_SIGNED + COMPARATOR_UNSIGNED + COMPARATOR_UNSIGNED_BYTES + COMPARATOR_DECIMAL + COMPARATOR_BOOLEAN + COMPARATOR_TYPE_FLOAT + COMPARATOR_IEEE_FLOAT + COMPARATOR_UNDEFINED +end + +struct ModelFormatError <: Exception + message::String +end + +function Base.showerror(io::IO, err::ModelFormatError) + print(io, err.message) + return +end + +struct LeafSpec + physical::PhysicalType + logical::LogicalType + type_length::Union{Nothing,Int} + bit_width::Union{Nothing,Int} + precision::Union{Nothing,Int} + time_unit::Union{Nothing,TimeUnit} +end + +function LeafSpec(physical::PhysicalType; logical::LogicalType=LOGICAL_NONE, + type_length::Union{Nothing,Integer}=nothing, + bit_width::Union{Nothing,Integer}=nothing, + precision::Union{Nothing,Integer}=nothing, + time_unit::Union{Nothing,TimeUnit}=nothing) + return LeafSpec(physical, logical, + type_length === nothing ? nothing : Int(type_length), + bit_width === nothing ? nothing : Int(bit_width), + precision === nothing ? nothing : Int(precision), time_unit) +end + +struct ModelLimits + max_statistics_value_bytes::Int64 +end + +function ModelLimits(; max_statistics_value_bytes::Integer=4096) + return ModelLimits(Int64(max_statistics_value_bytes)) +end + +abstract type ModelValue end + +struct SignedValue <: ModelValue + value::Int128 +end + +struct UnsignedValue <: ModelValue + value::UInt64 +end + +function UnsignedValue(value::Unsigned) + return UnsignedValue(UInt64(value)) +end + +struct BooleanValue <: ModelValue + value::Bool +end + +struct ByteValue <: ModelValue + value::Vector{UInt8} +end + +struct DecimalValue <: ModelValue + value::Vector{UInt8} +end + +struct FloatValue <: ModelValue + width::UInt8 + bits::UInt64 +end + +@enum BoundState begin + BOUND_ABSENT + BOUND_UNKNOWN + BOUND_KNOWN +end + +struct DecodedBound + state::BoundState + value::Union{Nothing,ModelValue} + reason::Symbol + semantic_checked::Bool +end + +function _known(value::ModelValue) + return DecodedBound(BOUND_KNOWN, value, :known, true) +end + +function _unknown(reason::Symbol, semantic_checked::Bool) + return DecodedBound(BOUND_UNKNOWN, nothing, reason, semantic_checked) +end + +function _readlittle(raw::AbstractVector{UInt8}) + value = UInt64(0) + offset = 0 + for byte in raw + value |= UInt64(byte) << offset + offset += 8 + end + return value +end + +function _signed(bits::UInt64, width::Int) + sign = UInt64(1) << (width - 1) + bits & sign == 0 && return Int128(bits) + return Int128(bits) - (Int128(1) << width) +end + +function _fixedwidth(leaf::LeafSpec) + leaf.physical == PHYSICAL_BOOLEAN && return 1 + leaf.physical in (PHYSICAL_INT32, PHYSICAL_FLOAT) && return 4 + leaf.physical in (PHYSICAL_INT64, PHYSICAL_DOUBLE) && return 8 + leaf.physical == PHYSICAL_INT96 && return 12 + if leaf.physical == PHYSICAL_FIXED_LEN_BYTE_ARRAY + leaf.type_length === nothing && throw(ModelFormatError( + "fixed byte-array leaf has no declared width")) + leaf.type_length >= 0 || throw(ModelFormatError( + "fixed byte-array leaf has a negative width")) + return leaf.type_length + end + return nothing +end + +function _checkstructure(raw::AbstractVector{UInt8}, leaf::LeafSpec) + width = _fixedwidth(leaf) + width === nothing && return + length(raw) == width || throw(ModelFormatError( + "fixed-width statistics bound has the wrong width")) + return +end + +function _validutf8(raw::AbstractVector{UInt8}) + return isvalid(String, raw) +end + +function _skipjsonspace(data::Vector{UInt8}, position::Int, stop::Int) + while position <= stop && data[position] in (0x20, 0x09, 0x0a, 0x0d) + position += 1 + end + return position +end + +function _jsonhex(byte::UInt8) + return (0x30 <= byte <= 0x39) || (0x41 <= byte <= 0x46) || + (0x61 <= byte <= 0x66) +end + +function _jsonstring(data::Vector{UInt8}, position::Int, stop::Int) + position <= stop && data[position] == 0x22 || return 0 + position += 1 + while position <= stop + byte = data[position] + byte == 0x22 && return position + 1 + byte < 0x20 && return 0 + if byte == 0x5c + position += 1 + position <= stop || return 0 + escape = data[position] + if escape == 0x75 + position + 4 <= stop || return 0 + for index in (position + 1):(position + 4) + _jsonhex(data[index]) || return 0 + end + position += 5 + continue + end + escape in (0x22, 0x5c, 0x2f, 0x62, 0x66, 0x6e, 0x72, 0x74) || + return 0 + end + position += 1 + end + return 0 +end + +function _jsondigits(data::Vector{UInt8}, position::Int, stop::Int) + start = position + while position <= stop && 0x30 <= data[position] <= 0x39 + position += 1 + end + return position == start ? 0 : position +end + +function _jsonnumber(data::Vector{UInt8}, position::Int, stop::Int) + position <= stop && data[position] == 0x2d && (position += 1) + position <= stop || return 0 + if data[position] == 0x30 + position += 1 + position <= stop && 0x30 <= data[position] <= 0x39 && return 0 + elseif 0x31 <= data[position] <= 0x39 + position = _jsondigits(data, position, stop) + else + return 0 + end + if position <= stop && data[position] == 0x2e + position = _jsondigits(data, position + 1, stop) + position == 0 && return 0 + end + if position <= stop && data[position] in (0x65, 0x45) + position += 1 + position <= stop && data[position] in (0x2b, 0x2d) && (position += 1) + position = _jsondigits(data, position, stop) + position == 0 && return 0 + end + return position +end + +function _jsonliteral(data::Vector{UInt8}, position::Int, stop::Int, + literal::String) + bytes = codeunits(literal) + position + length(bytes) - 1 <= stop || return 0 + for (offset, byte) in enumerate(bytes) + data[position + offset - 1] == byte || return 0 + end + return position + length(bytes) +end + +function _jsonarray(data::Vector{UInt8}, position::Int, stop::Int, depth::Int) + position = _skipjsonspace(data, position + 1, stop) + position <= stop && data[position] == 0x5d && return position + 1 + while position <= stop + position = _jsonvalue(data, position, stop, depth + 1) + position == 0 && return 0 + position = _skipjsonspace(data, position, stop) + position <= stop || return 0 + data[position] == 0x5d && return position + 1 + data[position] == 0x2c || return 0 + position = _skipjsonspace(data, position + 1, stop) + end + return 0 +end + +function _jsonobject(data::Vector{UInt8}, position::Int, stop::Int, depth::Int) + position = _skipjsonspace(data, position + 1, stop) + position <= stop && data[position] == 0x7d && return position + 1 + while position <= stop + position = _jsonstring(data, position, stop) + position == 0 && return 0 + position = _skipjsonspace(data, position, stop) + position <= stop && data[position] == 0x3a || return 0 + position = _skipjsonspace(data, position + 1, stop) + position = _jsonvalue(data, position, stop, depth + 1) + position == 0 && return 0 + position = _skipjsonspace(data, position, stop) + position <= stop || return 0 + data[position] == 0x7d && return position + 1 + data[position] == 0x2c || return 0 + position = _skipjsonspace(data, position + 1, stop) + end + return 0 +end + +function _jsonvalue(data::Vector{UInt8}, position::Int, stop::Int, depth::Int) + depth <= 64 || return 0 + position = _skipjsonspace(data, position, stop) + position <= stop || return 0 + byte = data[position] + byte == 0x22 && return _jsonstring(data, position, stop) + byte == 0x5b && return _jsonarray(data, position, stop, depth) + byte == 0x7b && return _jsonobject(data, position, stop, depth) + byte == 0x74 && return _jsonliteral(data, position, stop, "true") + byte == 0x66 && return _jsonliteral(data, position, stop, "false") + byte == 0x6e && return _jsonliteral(data, position, stop, "null") + return _jsonnumber(data, position, stop) +end + +function _validjson(data::Vector{UInt8}) + isempty(data) && return false + _validutf8(data) || return false + position = _jsonvalue(data, firstindex(data), lastindex(data), 0) + position == 0 && return false + return _skipjsonspace(data, position, lastindex(data)) == lastindex(data) + 1 +end + +function _readbsoni32(data::Vector{UInt8}, position::Int, stop::Int) + position + 3 <= stop || return nothing + bits = UInt32(data[position]) | (UInt32(data[position + 1]) << 8) | + (UInt32(data[position + 2]) << 16) | (UInt32(data[position + 3]) << 24) + value = bits <= UInt32(typemax(Int32)) ? Int(bits) : + Int(Int64(bits) - (Int64(1) << 32)) + return value +end + +function _bsoncstring(data::Vector{UInt8}, position::Int, stop::Int) + start = position + while position <= stop && data[position] != 0x00 + position += 1 + end + position <= stop || return 0 + _validutf8(view(data, start:(position - 1))) || return 0 + return position + 1 +end + +function _bsonregexoptions(data::Vector{UInt8}, position::Int, stop::Int) + previous = UInt8(0) + while position <= stop + byte = data[position] + byte == 0x00 && return position + 1 + byte in (0x69, 0x6d, 0x73, 0x75, 0x78) || return 0 + previous < byte || return 0 + previous = byte + position += 1 + end + return 0 +end + +function _bsonbinarysubtypevalid(subtype::UInt8) + return subtype <= 0x09 || subtype >= 0x80 +end + +function _bsonarraykey(data::Vector{UInt8}, position::Int, stop::Int, + index::Int) + expected = codeunits(string(index)) + position + length(expected) <= stop || return false + for (offset, byte) in enumerate(expected) + data[position + offset - 1] == byte || return false + end + return data[position + length(expected)] == 0x00 +end + +function _bsonbytes(position::Int, count::Int, stop::Int) + count >= 0 || return 0 + count <= stop - position + 1 || return 0 + return position + count +end + +function _bsonstring(data::Vector{UInt8}, position::Int, stop::Int) + count = _readbsoni32(data, position, stop) + count === nothing && return 0 + count >= 1 || return 0 + start = position + 4 + finish = _bsonbytes(start, count, stop) + finish == 0 && return 0 + data[finish - 1] == 0x00 || return 0 + _validutf8(view(data, start:(finish - 2))) || return 0 + return finish +end + +function _bsonvalue(data::Vector{UInt8}, position::Int, stop::Int, kind::UInt8, + depth::Int) + kind == 0x01 && return _bsonbytes(position, 8, stop) + kind == 0x02 && return _bsonstring(data, position, stop) + kind == 0x03 && return _bsondocument(data, position, stop, depth + 1, false) + kind == 0x04 && return _bsondocument(data, position, stop, depth + 1, true) + if kind == 0x05 + count = _readbsoni32(data, position, stop) + count === nothing && return 0 + count >= 0 || return 0 + position + 4 <= stop || return 0 + subtype = data[position + 4] + _bsonbinarysubtypevalid(subtype) || return 0 + payload = position + 5 + finish = _bsonbytes(payload, count, stop) + finish == 0 && return 0 + if subtype == 0x02 + count >= 4 || return 0 + oldcount = _readbsoni32(data, payload, finish - 1) + oldcount == count - 4 || return 0 + end + return finish + end + kind == 0x06 && return position + kind == 0x07 && return _bsonbytes(position, 12, stop) + if kind == 0x08 + position <= stop && data[position] in (0x00, 0x01) || return 0 + return position + 1 + end + kind == 0x09 && return _bsonbytes(position, 8, stop) + kind == 0x0a && return position + if kind == 0x0b + position = _bsoncstring(data, position, stop) + position == 0 && return 0 + return _bsonregexoptions(data, position, stop) + end + if kind == 0x0c + position = _bsonstring(data, position, stop) + position == 0 && return 0 + return _bsonbytes(position, 12, stop) + end + kind in (0x0d, 0x0e) && return _bsonstring(data, position, stop) + if kind == 0x0f + count = _readbsoni32(data, position, stop) + count === nothing && return 0 + count >= 14 || return 0 + finish = _bsonbytes(position, count, stop) + finish == 0 && return 0 + cursor = _bsonstring(data, position + 4, finish - 1) + cursor == 0 && return 0 + return _bsondocument(data, cursor, finish - 1, depth + 1, false) == + finish ? finish : 0 + end + kind == 0x10 && return _bsonbytes(position, 4, stop) + kind in (0x11, 0x12) && return _bsonbytes(position, 8, stop) + kind == 0x13 && return _bsonbytes(position, 16, stop) + kind in (0x7f, 0xff) && return position + return 0 +end + +function _bsondocument(data::Vector{UInt8}, position::Int, stop::Int, + depth::Int, isarray::Bool) + depth <= 64 || return 0 + count = _readbsoni32(data, position, stop) + count === nothing && return 0 + count >= 5 || return 0 + finish = _bsonbytes(position, count, stop) + finish == 0 && return 0 + data[finish - 1] == 0x00 || return 0 + cursor = position + 4 + index = 0 + while cursor < finish - 1 + kind = data[cursor] + isarray && !_bsonarraykey(data, cursor + 1, finish - 2, index) && return 0 + cursor = _bsoncstring(data, cursor + 1, finish - 2) + cursor == 0 && return 0 + cursor = _bsonvalue(data, cursor, finish - 2, kind, depth) + cursor == 0 && return 0 + index += 1 + end + return cursor == finish - 1 ? finish : 0 +end + +function _validbson(data::Vector{UInt8}) + isempty(data) && return false + return _bsondocument(data, firstindex(data), lastindex(data), 0, false) == + lastindex(data) + 1 +end + +function _normalizeddecimal(raw::Vector{UInt8}) + isempty(raw) && return UInt8[] + first = 1 + while first < length(raw) + byte = raw[first] + next = raw[first + 1] + byte == 0x00 && next < 0x80 && (first += 1; continue) + byte == 0xff && next >= 0x80 && (first += 1; continue) + break + end + return raw[first:end] +end + +function _stripzerobytes(raw::Vector{UInt8}) + isempty(raw) && return raw + first = findfirst(byte -> !iszero(byte), raw) + first === nothing && return UInt8[0x00] + return raw[first:end] +end + +function _decimalmagnitude(raw::Vector{UInt8}) + normalized = _normalizeddecimal(raw) + isempty(normalized) && return UInt8[] + normalized[1] < 0x80 && return _stripzerobytes(copy(normalized)) + magnitude = [~byte for byte in normalized] + carry = UInt16(1) + for index in lastindex(magnitude):-1:firstindex(magnitude) + value = UInt16(magnitude[index]) + carry + magnitude[index] = UInt8(value & 0xff) + carry = value >> 8 + iszero(carry) && break + end + return _stripzerobytes(magnitude) +end + +function _smallintdigits(value::UInt32) + digits = 1 + while value >= 10 + value ÷= 10 + digits += 1 + end + return digits +end + +function _decimaldigits(raw::Vector{UInt8}) + magnitude = _decimalmagnitude(raw) + isempty(magnitude) && return 0 + limbs = UInt32[0] + for byte in magnitude + carry = UInt64(byte) + for index in eachindex(limbs) + value = UInt64(limbs[index]) * 256 + carry + limbs[index] = UInt32(value % 1_000_000_000) + carry = value ÷ 1_000_000_000 + end + iszero(carry) || push!(limbs, UInt32(carry)) + end + while length(limbs) > 1 && iszero(last(limbs)) + pop!(limbs) + end + return (length(limbs) - 1) * 9 + _smallintdigits(last(limbs)) +end + +function _decimalvalue(raw::Vector{UInt8}, precision::Union{Nothing,Int}) + isempty(raw) && return nothing + precision === nothing && return nothing + precision > 0 || return nothing + _decimaldigits(raw) <= precision || return nothing + return DecimalValue(_normalizeddecimal(raw)) +end + +function _integerbytes(raw::Vector{UInt8}) + return _normalizeddecimal(reverse(raw)) +end + +function _integerlogical(raw::Vector{UInt8}, leaf::LeafSpec) + width = length(raw) * 8 + bits = _readlittle(raw) + if leaf.logical == LOGICAL_UNSIGNED_INTEGER + leaf.bit_width in (8, 16, 32, 64) || return nothing + leaf.bit_width <= width || return nothing + if leaf.bit_width < width + bits < (UInt64(1) << leaf.bit_width) || return nothing + end + return UnsignedValue(bits) + end + value = _signed(bits, width) + if leaf.logical == LOGICAL_SIGNED_INTEGER + leaf.bit_width in (8, 16, 32, 64) || return nothing + leaf.bit_width <= width || return nothing + low = -(Int128(1) << (leaf.bit_width - 1)) + high = (Int128(1) << (leaf.bit_width - 1)) - 1 + low <= value <= high || return nothing + end + return SignedValue(value) +end + +function _timevalid(value::Int128, unit::Union{Nothing,TimeUnit}) + unit === nothing && return false + limit = unit == TIME_MILLIS ? Int128(86_400_000) : + unit == TIME_MICROS ? Int128(86_400_000_000) : + Int128(86_400_000_000_000) + return 0 <= value < limit +end + +function _floatvalue(raw::Vector{UInt8}, width::Int) + return FloatValue(UInt8(width), _readlittle(raw)) +end + +function _logicalvalue(raw::Vector{UInt8}, leaf::LeafSpec) + logical = leaf.logical + if logical in (LOGICAL_STRING, LOGICAL_ENUM) + return _validutf8(raw) ? ByteValue(raw) : nothing + elseif logical == LOGICAL_JSON + return _validjson(raw) ? ByteValue(raw) : nothing + elseif logical == LOGICAL_BSON + return _validbson(raw) ? ByteValue(raw) : nothing + elseif logical == LOGICAL_UUID + length(raw) == 16 || return nothing + return ByteValue(raw) + elseif logical == LOGICAL_FLOAT16 + length(raw) == 2 || return nothing + return _floatvalue(raw, 16) + elseif logical == LOGICAL_DECIMAL + bytes = leaf.physical in (PHYSICAL_INT32, PHYSICAL_INT64) ? + _integerbytes(raw) : raw + return _decimalvalue(bytes, leaf.precision) + elseif logical in (LOGICAL_SIGNED_INTEGER, LOGICAL_UNSIGNED_INTEGER) + leaf.physical in (PHYSICAL_INT32, PHYSICAL_INT64) || return nothing + return _integerlogical(raw, leaf) + elseif logical in (LOGICAL_DATE, LOGICAL_TIMESTAMP) + leaf.physical in (PHYSICAL_INT32, PHYSICAL_INT64) || return nothing + return _integerlogical(raw, leaf) + elseif logical == LOGICAL_TIME + leaf.physical in (PHYSICAL_INT32, PHYSICAL_INT64) || return nothing + value = _integerlogical(raw, leaf) + value isa SignedValue || return nothing + return _timevalid(value.value, leaf.time_unit) ? value : nothing + end + leaf.physical == PHYSICAL_BOOLEAN && return raw[1] in (0x00, 0x01) ? + BooleanValue(raw[1] == 0x01) : nothing + leaf.physical in (PHYSICAL_INT32, PHYSICAL_INT64) && + return _integerlogical(raw, leaf) + leaf.physical == PHYSICAL_FLOAT && return _floatvalue(raw, 32) + leaf.physical == PHYSICAL_DOUBLE && return _floatvalue(raw, 64) + return ByteValue(raw) +end + +function decode_bound(raw::AbstractVector{UInt8}, leaf::LeafSpec, + limits::ModelLimits) + limits.max_statistics_value_bytes >= 0 || throw(ArgumentError( + "max_statistics_value_bytes must be nonnegative")) + _checkstructure(raw, leaf) + length(raw) > limits.max_statistics_value_bytes && + return _unknown(:over_limit, false) + bytes = Vector{UInt8}(raw) + value = _logicalvalue(bytes, leaf) + value === nothing && return _unknown(:invalid_logical, true) + return _known(value) +end + +function _compare(left, right) + left < right && return -1 + left > right && return 1 + return 0 +end + +function _comparebytes(left::Vector{UInt8}, right::Vector{UInt8}) + count = min(length(left), length(right)) + for index in 1:count + left[index] == right[index] && continue + return left[index] < right[index] ? -1 : 1 + end + return _compare(length(left), length(right)) +end + +function _decimalbyte(raw::Vector{UInt8}, offset::Int, width::Int) + padding = raw[1] >= 0x80 ? 0xff : 0x00 + skipped = width - length(raw) + return offset <= skipped ? padding : raw[offset - skipped] +end + +function _comparedecimal(left::DecimalValue, right::DecimalValue) + lnegative = left.value[1] >= 0x80 + rnegative = right.value[1] >= 0x80 + lnegative != rnegative && return lnegative ? -1 : 1 + width = max(length(left.value), length(right.value)) + for offset in 1:width + lbyte = _decimalbyte(left.value, offset, width) + rbyte = _decimalbyte(right.value, offset, width) + lbyte == rbyte && continue + return lbyte < rbyte ? -1 : 1 + end + return 0 +end + +function _floatmask(value::FloatValue) + value.width == 64 && return typemax(UInt64) + return (UInt64(1) << value.width) - 1 +end + +function _floatsignmask(value::FloatValue) + return UInt64(1) << (value.width - 1) +end + +function ieee_total_key(value::FloatValue) + mask = _floatmask(value) + bits = value.bits & mask + sign = _floatsignmask(value) + return bits & sign == 0 ? bits | sign : ~bits & mask +end + +function float_isnan(value::FloatValue) + if value.width == 16 + return value.bits & 0x7c00 == 0x7c00 && value.bits & 0x03ff != 0 + elseif value.width == 32 + return value.bits & 0x7f800000 == 0x7f800000 && + value.bits & 0x007fffff != 0 + elseif value.width == 64 + return value.bits & 0x7ff0000000000000 == 0x7ff0000000000000 && + value.bits & 0x000fffffffffffff != 0 + end + throw(ArgumentError("unsupported floating width")) +end + +function float_iszero(value::FloatValue) + mask = xor(_floatmask(value), _floatsignmask(value)) + return value.bits & mask == 0 +end + +function compare_values(left::ModelValue, right::ModelValue, + comparator::ComparatorKind) + if comparator == COMPARATOR_SIGNED + left isa SignedValue && right isa SignedValue || throw(ArgumentError( + "signed comparator requires signed values")) + return _compare(left.value, right.value) + elseif comparator == COMPARATOR_UNSIGNED + left isa UnsignedValue && right isa UnsignedValue || throw(ArgumentError( + "unsigned comparator requires unsigned values")) + return _compare(left.value, right.value) + elseif comparator == COMPARATOR_BOOLEAN + left isa BooleanValue && right isa BooleanValue || throw(ArgumentError( + "Boolean comparator requires Boolean values")) + return _compare(left.value, right.value) + elseif comparator == COMPARATOR_UNSIGNED_BYTES + left isa ByteValue && right isa ByteValue || throw(ArgumentError( + "byte comparator requires byte values")) + return _comparebytes(left.value, right.value) + elseif comparator == COMPARATOR_DECIMAL + left isa DecimalValue && right isa DecimalValue || throw(ArgumentError( + "decimal comparator requires decimal values")) + return _comparedecimal(left, right) + elseif comparator in (COMPARATOR_TYPE_FLOAT, COMPARATOR_IEEE_FLOAT) + left isa FloatValue && right isa FloatValue || throw(ArgumentError( + "floating comparator requires floating values")) + left.width == right.width || throw(ArgumentError( + "floating widths do not match")) + if comparator == COMPARATOR_TYPE_FLOAT + (float_isnan(left) || float_isnan(right)) && throw(ArgumentError( + "TYPE_ORDER cannot compare NaN bounds")) + float_iszero(left) && float_iszero(right) && return 0 + end + return _compare(ieee_total_key(left), ieee_total_key(right)) + end + throw(ArgumentError("undefined comparator")) +end + +function writer_bounds_allowed(lower::AbstractVector{UInt8}, + upper::AbstractVector{UInt8}, limits::ModelLimits) + limits.max_statistics_value_bytes >= 0 || throw(ArgumentError( + "max_statistics_value_bytes must be nonnegative")) + length(lower) <= limits.max_statistics_value_bytes || return false + return length(upper) <= limits.max_statistics_value_bytes +end + +@enum DeclaredOrder begin + ORDER_TYPE + ORDER_IEEE + ORDER_FUTURE +end + +@enum BoundFamily begin + FAMILY_NONE + FAMILY_MODERN + FAMILY_DEPRECATED +end + +@enum Exactness begin + EXACTNESS_UNKNOWN + EXACTNESS_INEXACT + EXACTNESS_EXACT +end + +@enum OccupancyState begin + OCCUPANCY_UNKNOWN + OCCUPANCY_EMPTY + OCCUPANCY_ALL_NAN + OCCUPANCY_HAS_NON_NAN +end + +@enum TrustState begin + TRUST_TRUSTED + TRUST_UNTRUSTED +end + +Base.@kwdef struct RawStatistics + modern_lower::Union{Nothing,Vector{UInt8}} = nothing + modern_upper::Union{Nothing,Vector{UInt8}} = nothing + deprecated_lower::Union{Nothing,Vector{UInt8}} = nothing + deprecated_upper::Union{Nothing,Vector{UInt8}} = nothing + null_count::Union{Nothing,Int64} = nothing + nan_count::Union{Nothing,Int64} = nothing + distinct_count::Union{Nothing,Int64} = nothing + lower_exact::Union{Nothing,Bool} = nothing + upper_exact::Union{Nothing,Bool} = nothing +end + +struct CountFact + known::Bool + value::Int64 +end + +struct BoundFact + state::BoundState + value::Union{Nothing,ModelValue} + raw::Union{Nothing,Vector{UInt8}} + exactness::Exactness + reason::Symbol +end + +struct TrustDecision + state::TrustState + reason::Symbol +end + +struct StatisticsResult + lower::BoundFact + upper::BoundFact + null_count::CountFact + nan_count::CountFact + distinct_count::CountFact + occupancy::OccupancyState + family::BoundFamily + comparator::ComparatorKind + trust::TrustDecision +end + +struct ExtremaSummary + lower::Union{Nothing,ModelValue} + upper::Union{Nothing,ModelValue} + nan_count::Int64 + value_count::Int64 +end + +struct SemanticVersion + major::Int + minor::Int + patch::Int + unknown::String + has_prerelease::Bool + prerelease::Vector{String} +end + +struct ParsedProducer + parsed::Bool + application::String + version::Union{Nothing,SemanticVersion} +end + +function _javadotsplit(label::AbstractString) + isempty(label) && return [""] + identifiers = String.(split(label, '.'; keepempty=true)) + while !isempty(identifiers) && isempty(last(identifiers)) + pop!(identifiers) + end + return identifiers +end + +function _javaisspace(character::Char) + return character in (' ', '\t', '\n', '\v', '\f', '\r') +end + +function _javaasciistrip(text::AbstractString) + return strip(_javaisspace, text) +end + +function _javacontainslineterminator(text::AbstractString) + return any(character -> character in ('\n', '\r', '\u0085', '\u2028', + '\u2029'), text) +end + +function _semver(text::AbstractString) + matched = match(r"^([0-9]+)\.([0-9]+)\.([0-9]+)([^-+]*)?(?:-([^+]*))?(?:\+(.*))?$", + text) + matched === nothing && return nothing + build = matched.captures[6] + build !== nothing && _javacontainslineterminator(build) && return nothing + major = tryparse(Int32, matched.captures[1]) + minor = tryparse(Int32, matched.captures[2]) + patch = tryparse(Int32, matched.captures[3]) + any(isnothing, (major, minor, patch)) && return nothing + unknown = something(matched.captures[4], "") + label = matched.captures[5] + prerelease = label === nothing ? String[] : + _javadotsplit(label) + for identifier in prerelease + occursin(r"^[0-9]+$", identifier) || continue + tryparse(Int32, identifier) === nothing && return nothing + end + return SemanticVersion(Int(major), Int(minor), Int(patch), unknown, + label !== nothing, prerelease) +end + +function parse_created_by(created_by::Union{Nothing,AbstractString}) + created_by === nothing && return ParsedProducer(false, "", nothing) + text = _javaasciistrip(String(created_by)) + isempty(text) && return ParsedProducer(false, "", nothing) + matched = match( + r"^(.*?)[ \t\n\x0b\f\r]+version[ \t\n\x0b\f\r]*(?:([^(]*?)[ \t\n\x0b\f\r]*(?:\([ \t\n\x0b\f\r]*build[ \t\n\x0b\f\r]*([^)]*?)[ \t\n\x0b\f\r]*\))?)?$", + text) + if matched !== nothing + application = _javaasciistrip(matched.captures[1]) + isempty(application) && return ParsedProducer(false, "", nothing) + _javacontainslineterminator(application) && + return ParsedProducer(false, "", nothing) + rawversion = matched.captures[2] + versiontext = rawversion === nothing ? "" : _javaasciistrip(rawversion) + version = isempty(versiontext) ? nothing : _semver(versiontext) + return ParsedProducer(true, application, version) + end + return ParsedProducer(false, "", nothing) +end + +function parse_arrow_created_by(created_by::Union{Nothing,AbstractString}) + producer = parse_created_by(created_by) + producer.parsed && return producer + created_by === nothing && return producer + application = _javaasciistrip(String(created_by)) + application in ("parquet-cpp", "parquet-mr") || return producer + return ParsedProducer(true, application, nothing) +end + +function _identifiercompare(left::String, right::String) + lnumber = occursin(r"^[0-9]+$", left) ? tryparse(Int32, left) : nothing + rnumber = occursin(r"^[0-9]+$", right) ? tryparse(Int32, right) : nothing + lnumber !== nothing && rnumber !== nothing && return _compare(lnumber, rnumber) + lnumber !== nothing && return -1 + rnumber !== nothing && return 1 + return _compare(left, right) +end + +function _prereleasecompare(left::Vector{String}, right::Vector{String}) + for index in 1:min(length(left), length(right)) + compared = _identifiercompare(left[index], right[index]) + iszero(compared) || return compared + end + return _compare(length(left), length(right)) +end + +function _versioncompare(left::SemanticVersion, right::SemanticVersion) + compared = _compare(left.major, right.major) + iszero(compared) || return compared + compared = _compare(left.minor, right.minor) + iszero(compared) || return compared + compared = _compare(left.patch, right.patch) + iszero(compared) || return compared + lunknown = !isempty(left.unknown) + runknown = !isempty(right.unknown) + lunknown != runknown && return lunknown ? -1 : 1 + left.has_prerelease != right.has_prerelease && + return left.has_prerelease ? -1 : 1 + return _prereleasecompare(left.prerelease, right.prerelease) +end + +function _versionlt(left::SemanticVersion, right::SemanticVersion) + return _versioncompare(left, right) < 0 +end + +function _parquet251affected(producer::ParsedProducer) + producer.parsed || return true + producer.application == "parquet-mr" || return false + producer.version === nothing && return true + fixed = SemanticVersion(1, 8, 0, "", false, String[]) + _versionlt(producer.version, fixed) || return false + cdhstart = SemanticVersion(1, 5, 0, "", true, ["cdh5", "5", "0"]) + cdhend = SemanticVersion(1, 5, 0, "", false, String[]) + incdh = !_versionlt(producer.version, cdhstart) && + _versionlt(producer.version, cdhend) + return !incdh +end + +function _oldorderaffected(producer::ParsedProducer) + cutoff = if producer.application == "parquet-cpp" + SemanticVersion(1, 3, 0, "", false, String[]) + elseif producer.application == "parquet-mr" + SemanticVersion(1, 10, 0, "", false, String[]) + else + return false + end + producer.version === nothing && return true + return _versionlt(producer.version, cutoff) +end + +function _legacyorderissigned(comparator::ComparatorKind) + return comparator in (COMPARATOR_SIGNED, COMPARATOR_BOOLEAN, + COMPARATOR_DECIMAL, COMPARATOR_TYPE_FLOAT) +end + +function _binaryleaf(leaf::LeafSpec) + return leaf.physical in (PHYSICAL_BYTE_ARRAY, PHYSICAL_FIXED_LEN_BYTE_ARRAY) +end + +function producer_decision(created_by::Union{Nothing,AbstractString}, + leaf::LeafSpec, comparator::ComparatorKind, family::BoundFamily, + lower::Union{Nothing,AbstractVector{UInt8}}, + upper::Union{Nothing,AbstractVector{UInt8}}, + limits::ModelLimits=ModelLimits()) + limits.max_statistics_value_bytes >= 0 || throw(ArgumentError( + "max_statistics_value_bytes must be nonnegative")) + family == FAMILY_NONE && return TrustDecision(TRUST_TRUSTED, :no_bounds) + java_producer = parse_created_by(created_by) + if _binaryleaf(leaf) && _parquet251affected(java_producer) + return TrustDecision(TRUST_UNTRUSTED, :parquet_251) + end + arrow_producer = parse_arrow_created_by(created_by) + if _oldorderaffected(arrow_producer) && !_legacyorderissigned(comparator) + equal = lower !== nothing && upper !== nothing && + length(lower) <= limits.max_statistics_value_bytes && + length(upper) <= limits.max_statistics_value_bytes && lower == upper + equal || return TrustDecision(TRUST_UNTRUSTED, :legacy_wrong_order) + end + return TrustDecision(TRUST_TRUSTED, :trusted) +end + +function _floatleaf(leaf::LeafSpec) + return leaf.physical in (PHYSICAL_FLOAT, PHYSICAL_DOUBLE) || + leaf.logical == LOGICAL_FLOAT16 +end + +function _typecomparator(leaf::LeafSpec) + leaf.logical in (LOGICAL_INTERVAL, LOGICAL_UNKNOWN, LOGICAL_VARIANT, + LOGICAL_GEOMETRY, LOGICAL_GEOGRAPHY, LOGICAL_LIST, LOGICAL_MAP) && + return COMPARATOR_UNDEFINED + leaf.physical == PHYSICAL_INT96 && return COMPARATOR_UNDEFINED + leaf.logical == LOGICAL_UNSIGNED_INTEGER && return COMPARATOR_UNSIGNED + leaf.logical in (LOGICAL_STRING, LOGICAL_ENUM, LOGICAL_JSON, LOGICAL_BSON, + LOGICAL_UUID) && return COMPARATOR_UNSIGNED_BYTES + leaf.logical == LOGICAL_DECIMAL && return COMPARATOR_DECIMAL + leaf.logical == LOGICAL_FLOAT16 && return COMPARATOR_TYPE_FLOAT + leaf.logical in (LOGICAL_SIGNED_INTEGER, LOGICAL_DATE, LOGICAL_TIME, + LOGICAL_TIMESTAMP) && return COMPARATOR_SIGNED + leaf.physical == PHYSICAL_BOOLEAN && return COMPARATOR_BOOLEAN + leaf.physical in (PHYSICAL_INT32, PHYSICAL_INT64) && return COMPARATOR_SIGNED + leaf.physical in (PHYSICAL_FLOAT, PHYSICAL_DOUBLE) && + return COMPARATOR_TYPE_FLOAT + leaf.physical in (PHYSICAL_BYTE_ARRAY, PHYSICAL_FIXED_LEN_BYTE_ARRAY) && + return COMPARATOR_UNSIGNED_BYTES + return COMPARATOR_UNDEFINED +end + +function _deprecatedcompatible(comparator::ComparatorKind) + return _legacyorderissigned(comparator) +end + +function _countfact(value::Union{Nothing,Int64}, num_values::Int64, + name::String) + value === nothing && return CountFact(false, Int64(0)) + 0 <= value <= num_values || throw(ModelFormatError( + "$name is outside the column value count")) + return CountFact(true, value) +end + +function _counts(leaf::LeafSpec, num_values::Int64, stats::RawStatistics) + num_values >= 0 || throw(ModelFormatError("column value count is negative")) + nulls = _countfact(stats.null_count, num_values, "null_count") + nans = _countfact(stats.nan_count, num_values, "nan_count") + distinct = _countfact(stats.distinct_count, num_values, "distinct_count") + nans.known && !_floatleaf(leaf) && throw(ModelFormatError( + "nan_count is present on a non-floating leaf")) + if nulls.known && nans.known + nulls.value <= num_values - nans.value || throw(ModelFormatError( + "null_count plus nan_count exceeds num_values")) + end + if nulls.known && distinct.known + distinct.value <= num_values - nulls.value || throw(ModelFormatError( + "distinct_count exceeds the non-null value count")) + end + occupancy = if iszero(num_values) + OCCUPANCY_EMPTY + elseif nulls.known && nulls.value == num_values + OCCUPANCY_EMPTY + elseif nulls.known && nans.known + nonnull = num_values - nulls.value + nonnull > 0 && nans.value == nonnull ? OCCUPANCY_ALL_NAN : + OCCUPANCY_HAS_NON_NAN + else + OCCUPANCY_UNKNOWN + end + return nulls, nans, distinct, occupancy +end + +function _selectedfamily(stats::RawStatistics) + (stats.modern_lower !== nothing || stats.modern_upper !== nothing) && + return FAMILY_MODERN + (stats.deprecated_lower !== nothing || stats.deprecated_upper !== nothing) && + return FAMILY_DEPRECATED + return FAMILY_NONE +end + +function _selectedraw(stats::RawStatistics, family::BoundFamily) + family == FAMILY_MODERN && return stats.modern_lower, stats.modern_upper + family == FAMILY_DEPRECATED && + return stats.deprecated_lower, stats.deprecated_upper + return nothing, nothing +end + +function _exactness(flag::Union{Nothing,Bool}, family::BoundFamily) + family == FAMILY_DEPRECATED && return EXACTNESS_UNKNOWN + flag === nothing && return EXACTNESS_UNKNOWN + return flag ? EXACTNESS_EXACT : EXACTNESS_INEXACT +end + +function _absentbound() + return BoundFact(BOUND_ABSENT, nothing, nothing, EXACTNESS_UNKNOWN, :absent) +end + +function _boundfact(raw::Union{Nothing,Vector{UInt8}}, leaf::LeafSpec, + limits::ModelLimits, exactness::Exactness) + raw === nothing && return _absentbound() + decoded = decode_bound(raw, leaf, limits) + return BoundFact(decoded.state, decoded.value, raw, exactness, decoded.reason) +end + +function _invalidate(bound::BoundFact, reason::Symbol) + bound.state != BOUND_KNOWN && return bound + return BoundFact(BOUND_UNKNOWN, nothing, bound.raw, bound.exactness, reason) +end + +function _invalidateboth(lower::BoundFact, upper::BoundFact, reason::Symbol) + return _invalidate(lower, reason), _invalidate(upper, reason) +end + +function _validateorders(orders::Union{Nothing,Vector{DeclaredOrder}}, + leaf::LeafSpec, leaf_index::Int, leaf_count::Int) + orders === nothing && return nothing + length(orders) == leaf_count || throw(ModelFormatError( + "column_orders is not leaf aligned")) + 1 <= leaf_index <= leaf_count || throw(ArgumentError( + "leaf index is outside the schema")) + order = orders[leaf_index] + order == ORDER_IEEE && !_floatleaf(leaf) && throw(ModelFormatError( + "IEEE total order is present on a non-floating leaf")) + return order +end + +function _comparator(leaf::LeafSpec, family::BoundFamily, + order::Union{Nothing,DeclaredOrder}) + typecomparator = _typecomparator(leaf) + if family == FAMILY_MODERN + order === nothing && return COMPARATOR_UNDEFINED, :missing_column_orders + order == ORDER_FUTURE && return COMPARATOR_UNDEFINED, :unknown_column_order + order == ORDER_IEEE && return COMPARATOR_IEEE_FLOAT, :known + typecomparator == COMPARATOR_UNDEFINED && + return COMPARATOR_UNDEFINED, :undefined_type_order + return typecomparator, :known + elseif family == FAMILY_DEPRECATED + _deprecatedcompatible(typecomparator) || + return COMPARATOR_UNDEFINED, :deprecated_order_mismatch + return typecomparator, :known + end + return COMPARATOR_UNDEFINED, :no_bounds +end + +function _floatnegative(value::FloatValue) + return value.bits & _floatsignmask(value) != 0 +end + +function _widenzero(bound::BoundFact, lower::Bool) + bound.state == BOUND_KNOWN || return bound + value = bound.value + value isa FloatValue || return bound + float_iszero(value) || return bound + needs = lower ? !_floatnegative(value) : _floatnegative(value) + needs || return bound + bits = lower ? value.bits | _floatsignmask(value) : + value.bits & xor(_floatmask(value), _floatsignmask(value)) + exactness = bound.exactness == EXACTNESS_EXACT ? EXACTNESS_INEXACT : + bound.exactness + return BoundFact(BOUND_KNOWN, FloatValue(value.width, bits), bound.raw, + exactness, :widened_zero) +end + +function _boundisnan(bound::BoundFact) + return bound.state == BOUND_KNOWN && bound.value isa FloatValue && + float_isnan(bound.value) +end + +function _applyfloatrules(lower::BoundFact, upper::BoundFact, + comparator::ComparatorKind, occupancy::OccupancyState) + if comparator == COMPARATOR_TYPE_FLOAT + if occupancy == OCCUPANCY_ALL_NAN && + (lower.state != BOUND_ABSENT || upper.state != BOUND_ABSENT) + return _invalidateboth(lower, upper, :all_nan_type_order) + end + _boundisnan(lower) && (lower = _invalidate(lower, :nan_type_order)) + _boundisnan(upper) && (upper = _invalidate(upper, :nan_type_order)) + lower = _widenzero(lower, true) + upper = _widenzero(upper, false) + return lower, upper + elseif comparator == COMPARATOR_IEEE_FLOAT + if occupancy == OCCUPANCY_ALL_NAN + contradiction = (lower.state == BOUND_KNOWN && !_boundisnan(lower)) || + (upper.state == BOUND_KNOWN && !_boundisnan(upper)) + contradiction && return _invalidateboth(lower, upper, + :ieee_bound_kind_contradiction) + elseif occupancy == OCCUPANCY_HAS_NON_NAN + (_boundisnan(lower) || _boundisnan(upper)) && + return _invalidateboth(lower, upper, + :ieee_bound_kind_contradiction) + else + _boundisnan(lower) && + (lower = _invalidate(lower, :unproven_ieee_nan)) + _boundisnan(upper) && + (upper = _invalidate(upper, :unproven_ieee_nan)) + end + end + return lower, upper +end + +function _checkboundorder(lower::BoundFact, upper::BoundFact, + comparator::ComparatorKind) + lower.state == BOUND_KNOWN && upper.state == BOUND_KNOWN || + return lower, upper + compare_values(lower.value, upper.value, comparator) <= 0 && + return lower, upper + return _invalidateboth(lower, upper, :contradictory_bounds) +end + +function interpret_statistics(leaf::LeafSpec, num_values::Int64, + stats::RawStatistics, orders::Union{Nothing,Vector{DeclaredOrder}}; + leaf_index::Int=1, leaf_count::Int=1, + created_by::Union{Nothing,AbstractString}=nothing, + limits::ModelLimits=ModelLimits()) + limits.max_statistics_value_bytes >= 0 || throw(ArgumentError( + "max_statistics_value_bytes must be nonnegative")) + nulls, nans, distinct, occupancy = _counts(leaf, num_values, stats) + order = _validateorders(orders, leaf, leaf_index, leaf_count) + family = _selectedfamily(stats) + lowerraw, upperraw = _selectedraw(stats, family) + lowerraw === nothing || _checkstructure(lowerraw, leaf) + upperraw === nothing || _checkstructure(upperraw, leaf) + comparator, orderreason = _comparator(leaf, family, order) + lower = _boundfact(lowerraw, leaf, limits, + _exactness(stats.lower_exact, family)) + upper = _boundfact(upperraw, leaf, limits, + _exactness(stats.upper_exact, family)) + trust = producer_decision(created_by, leaf, comparator, family, lowerraw, + upperraw, limits) + if orderreason != :known && family != FAMILY_NONE + lower, upper = _invalidateboth(lower, upper, orderreason) + elseif trust.state == TRUST_UNTRUSTED + lower, upper = _invalidateboth(lower, upper, trust.reason) + elseif occupancy == OCCUPANCY_EMPTY + lower, upper = _invalidateboth(lower, upper, :no_non_null_values) + elseif comparator in (COMPARATOR_TYPE_FLOAT, COMPARATOR_IEEE_FLOAT) + lower, upper = _applyfloatrules(lower, upper, comparator, occupancy) + end + if comparator != COMPARATOR_UNDEFINED + lower, upper = _checkboundorder(lower, upper, comparator) + end + return StatisticsResult(lower, upper, nulls, nans, distinct, occupancy, + family, comparator, trust) +end + +function _summaryvalue(raw::AbstractVector{UInt8}, leaf::LeafSpec) + _checkstructure(raw, leaf) + value = _logicalvalue(Vector{UInt8}(raw), leaf) + value === nothing && throw(ModelFormatError( + "summary input is not a valid logical value")) + return value +end + +function _summarycandidate(value::ModelValue, comparator::ComparatorKind, + has_non_nan::Bool) + value isa FloatValue || return true + float_isnan(value) || return true + comparator == COMPARATOR_TYPE_FLOAT && return false + return comparator == COMPARATOR_IEEE_FLOAT && !has_non_nan +end + +function _summaryzeros(lower::ModelValue, upper::ModelValue, + comparator::ComparatorKind) + comparator == COMPARATOR_TYPE_FLOAT || return lower, upper + lower isa FloatValue && upper isa FloatValue || return lower, upper + if float_iszero(lower) + lower = FloatValue(lower.width, lower.bits | _floatsignmask(lower)) + end + if float_iszero(upper) + upper = FloatValue(upper.width, + upper.bits & xor(_floatmask(upper), _floatsignmask(upper))) + end + return lower, upper +end + +function summarize_raw_values(leaf::LeafSpec, + raw_values::AbstractVector{<:AbstractVector{UInt8}}, + comparator::ComparatorKind) + values = ModelValue[] + sizehint!(values, length(raw_values)) + nan_count = Int64(0) + has_non_nan = false + for raw in raw_values + value = _summaryvalue(raw, leaf) + push!(values, value) + if value isa FloatValue && float_isnan(value) + nan_count += 1 + else + has_non_nan = true + end + end + lower = nothing + upper = nothing + for value in values + _summarycandidate(value, comparator, has_non_nan) || continue + if lower === nothing + lower = value + upper = value + continue + end + compare_values(value, lower, comparator) < 0 && (lower = value) + compare_values(value, upper, comparator) > 0 && (upper = value) + end + if lower !== nothing + lower, upper = _summaryzeros(lower, upper, comparator) + end + return ExtremaSummary(lower, upper, nan_count, Int64(length(values))) +end + +function contains_value(summary::ExtremaSummary, value::ModelValue, + comparator::ComparatorKind) + summary.lower === nothing && return false + summary.upper === nothing && return false + if value isa FloatValue + lowerisnan = summary.lower isa FloatValue && float_isnan(summary.lower) + upperisnan = summary.upper isa FloatValue && float_isnan(summary.upper) + valueisnan = float_isnan(value) + if comparator == COMPARATOR_IEEE_FLOAT + lowerisnan == upperisnan || return false + valueisnan == lowerisnan || return false + else + valueisnan && return false + end + end + compare_values(summary.lower, value, comparator) <= 0 || return false + return compare_values(value, summary.upper, comparator) <= 0 +end + +end diff --git a/test/conformance/n6/model/README.md b/test/conformance/n6/model/README.md new file mode 100644 index 0000000..891c170 --- /dev/null +++ b/test/conformance/n6/model/README.md @@ -0,0 +1,49 @@ +# Independent N6 statistics model + +This directory contains the frozen test-only semantic model for N6-A. Its authority is +`docs/dev/n6-statistics-plan.md` at SHA-256 +`15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304`. + +The module owns its metadata-like types, PLAIN bound decoding, logical validation, +decimal comparison, producer parser and trust policy, count state machine, bound-family +selection, TYPE_ORDER compatibility, and raw-bit IEEE total order. It does not load the +Parquet package. It does not call a production decoder or comparator. The tests scan the +Julia files in this directory to keep that boundary explicit. + +`LeafSpec` represents a leaf from an already validated Parquet schema. The model does +not validate whether a logical type and physical type may be paired in a schema. It +validates statistics widths, values, order, counts, and trust after that schema gate. + +Producer semantic versions must contain `major.minor.patch`. Apache Arrow's policy +parser defaults omitted minor or patch components to zero. This model deliberately does +not. Partial versions such as `1`, `1.2`, `1.3`, and `1.10` have no usable version. A +recognized parquet-cpp or parquet-mr producer with such a version is conservatively in +the affected range for the old non-signed-order rule. + +The compatibility parsers are separate. `parse_created_by` follows pinned parquet-java +and rejects bare application tokens. The Arrow old-order parser additionally recognizes +bare `parquet-cpp` and `parquet-mr` as those applications with no usable version. It does +not recognize other bare text. PARQUET-251 uses only the parquet-java parser, so every +bare or wholly unparsable value is untrusted on byte-array leaves. PARQUET-251 runs +before the Arrow equality exception. These policies may discard bounds, but cannot +create a false exclusion. + +When `num_values` is zero, occupancy is empty without requiring optional count fields. +Any present bounds then become unknown. + +Run the model directly from the repository root: + +```sh +julia --startup-file=no test/conformance/n6/model/runtests.jl +``` + +`cases.toml` freezes the format, parquet-java, and Arrow policy source pins. It also +freezes the coverage groups, limit rules, and exhaustive Float16 order digest. The +digest covers every UInt16 bit pattern serialized in little-endian order after IEEE +total-order sorting. + +This model interprets already-extracted statistics fields and computes independent +extrema from raw scalar values. It does not decode Compact Thrift or inspect Parquet +files. The separately pinned non-Julia raw 2.13 scanner owns that evidence. This model +also does not authorize pruning, ColumnIndex content, an oracle image, or +`oracles.lock`. diff --git a/test/conformance/n6/model/cases.toml b/test/conformance/n6/model/cases.toml new file mode 100644 index 0000000..a2b57f3 --- /dev/null +++ b/test/conformance/n6/model/cases.toml @@ -0,0 +1,114 @@ +schema_version = 1 +authority_plan_sha256 = "15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304" +parquet_format_commit = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" +arrow_statistics_policy_commit = "515410b2a14ac766258e00b07eab9e5ee2692a62" +parquet_java_tag = "apache-parquet-1.17.1" +parquet_java_commit = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +parquet_java_embedded_format = "2.12" +module = "N6StatisticsModel.jl" +runner = "runtests.jl" +production_dependency_allowed = false + +[float16] +pattern_count = 65536 +nan_pattern_count = 2046 +ordered_little_endian_sha256 = "61619b1a4260ee4cff6d462d21cab5a049198a27b3a9ae3e8ecfe246efc1d4b6" +first_pattern = "ffff" +last_pattern = "7fff" + +[limits] +default_statistics_value_bytes = 4096 +equality_succeeds = true +fixed_structure_precedes_limit = true +variable_limit_precedes_semantics = true + +[[case_groups]] +id = "plain-bound-decoding" +requirements = ["exact fixed widths", "little-endian scalars", "raw variable bytes"] +capabilities = ["semantic.type-order"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.type-order" = "15a1da9c96f3c7cf71dc3a61022aa49bc9b6c235f22288fd0340c460f728eee1" } + +[[case_groups]] +id = "logical-bound-validity" +requirements = ["UTF-8", "JSON", "BSON structure", "BSON binary subtype", "BSON regex options", "UUID", "INTEGER", "TIME", "FLOAT16"] +capabilities = ["semantic.logical-order"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.logical-order" = "59acacd3241521be47618a40f279fb36bd4141e2c971f510551b6e7d8b34ed53" } + +[[case_groups]] +id = "decimal-order" +requirements = ["two's-complement normalization", "precision", "no BigInt"] +capabilities = ["semantic.logical-order"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.logical-order" = "b9fc8ce26cbbb5f44922aa90053520249f7d2e35a566740ebfd7d3436dfdaf1a" } + +[[case_groups]] +id = "count-state-machine" +requirements = ["null", "NaN", "distinct", "checked relationships", "zero values without counts", "empty bounds", "all-NaN"] +capabilities = ["semantic.count-state", "wire.statistics.nan-count"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.count-state" = "d8a7163f59defdcb8c771e07c0ddec65e873967340355e2eda07f63129eeff01", "wire.statistics.nan-count" = "123d4393ed7f2b9df6b22642f7e293a1db6e4c7ffac4b5ab30b3e7b34ec8acbe" } + +[[case_groups]] +id = "atomic-bound-family" +requirements = ["modern precedence", "no deprecated fill", "one-sided bounds"] +capabilities = ["semantic.count-state", "semantic.type-order"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.count-state" = "ce7d5eba1d6c8167af6d8c78b753c76dda1adcbdce93427569bca80334ea054d", "semantic.type-order" = "03356ecaa321e342f7362031499f4a110c6d426619b3abcea19bd869f6f38fb4" } + +[[case_groups]] +id = "type-order-float" +requirements = ["independent NaN rejection", "signed-zero widening", "exactness downgrade"] +capabilities = ["semantic.type-order"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.type-order" = "fb4ad657bd0f65b0e9b6b0c470e4290994724f60ef06774146c3d0b7777b7023" } + +[[case_groups]] +id = "ieee-total-order" +requirements = ["raw-bit key", "NaN payloads", "signed zeros", "count-bound states"] +capabilities = ["semantic.ieee-total-order", "wire.statistics.nan-count"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.ieee-total-order" = "d596fee1637b66a94cef82ad91332ef69393fa486400ea8ee7cdbcde2ac97ccd", "wire.statistics.nan-count" = "2b167a8c7168592a836b8abc6856d15440bf2c69a365c05dd3f896eb4750628a" } + +[[case_groups]] +id = "float16-exhaustive" +requirements = ["all 65536 patterns", "all 2046 NaNs", "frozen order digest"] +capabilities = ["semantic.ieee-total-order"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.ieee-total-order" = "e4d2981184aa5082519114ba3dc7e5f000fdf0e70067fe33750dbe6ac7c6717b" } + +[[case_groups]] +id = "producer-parquet-251" +requirements = ["parquet-java parser", "bare token rejection", "both families", "CDH exception", "RC", "final", "partial version", "equality does not override"] +capabilities = ["compat.legacy-statistics", "semantic.producer-trust"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "compat.legacy-statistics" = "cdaa409cc6dada3fada0752543aa6911174cd2523a38f996490e814bc1f58b1a", "semantic.producer-trust" = "fb6f35682ebf84d6b3bb9a505cc91dd6154c8f5c05fab20798081afb11d1f496" } + +[[case_groups]] +id = "producer-old-order" +requirements = ["Arrow parser", "bare parquet producer", "parquet-cpp 1.3", "parquet-mr 1.10", "RC", "final", "partial versions conservative", "IEEE non-signed", "equal bytes"] +capabilities = ["semantic.producer-trust"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.producer-trust" = "689dc5b2feee3b711beb804acd48d1c1b47f6d5654e94bccba1d7429154dd91d" } + +[[case_groups]] +id = "limit-precedence" +requirements = ["negative entry failure", "zero", "exact", "one over", "writer omit both"] +capabilities = [] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = {} + +[[case_groups]] +id = "independent-extrema" +requirements = ["mixed NaN", "all NaN", "TYPE_ORDER", "IEEE order", "containment", "input snapshot"] +capabilities = ["semantic.ieee-total-order"] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = { "semantic.ieee-total-order" = "f7e06eaadeda5d00e52ed4f7196cb3e39ce867fd256814bf8876ad165cfb1288" } + +[[case_groups]] +id = "forbidden-dependencies" +requirements = ["no package import", "no production source include", "no production namespace call"] +capabilities = [] +digest_contract = "n6-capability-result-sha256-v1" +expected_sha256 = {} diff --git a/test/conformance/n6/model/runtests.jl b/test/conformance/n6/model/runtests.jl new file mode 100644 index 0000000..98325c4 --- /dev/null +++ b/test/conformance/n6/model/runtests.jl @@ -0,0 +1,920 @@ +using SHA +using Test +using TOML + +include("N6StatisticsModel.jl") +const Model = N6StatisticsModel + +function le32(bits::UInt32) + return UInt8[bits & 0xff, (bits >> 8) & 0xff, (bits >> 16) & 0xff, + (bits >> 24) & 0xff] +end + +function le16(bits::UInt16) + return UInt8[bits & 0xff, (bits >> 8) & 0xff] +end + +function be16(value::Int16) + bits = reinterpret(UInt16, value) + return UInt8[bits >> 8, bits & 0xff] +end + +function bsonbinary(subtype::UInt8) + return UInt8[0x0e, 0x00, 0x00, 0x00, 0x05, 0x62, 0x00, + 0x01, 0x00, 0x00, 0x00, subtype, 0x61, 0x00] +end + +function bsonregex(options::AbstractString) + optionbytes = Vector{UInt8}(codeunits(options)) + length = 11 + Base.length(optionbytes) + return vcat(le32(UInt32(length)), UInt8[0x0b, 0x72, 0x00, 0x61, 0x00], + optionbytes, UInt8[0x00, 0x00]) +end + +@testset "N6 independent statistics model" begin + @testset "signed PLAIN bound decoding" begin + leaf = Model.LeafSpec(Model.PHYSICAL_INT32) + decoded = Model.decode_bound(UInt8[0xf9, 0xff, 0xff, 0xff], leaf, + Model.ModelLimits()) + @test decoded.state == Model.BOUND_KNOWN + @test decoded.value == Model.SignedValue(Int128(-7)) + end + + @testset "physical and logical bound decoding" begin + @test Model.decode_bound(UInt8[0x01], + Model.LeafSpec(Model.PHYSICAL_BOOLEAN), Model.ModelLimits()).value == + Model.BooleanValue(true) + @test_throws Model.ModelFormatError Model.decode_bound(UInt8[0x01, 0x00], + Model.LeafSpec(Model.PHYSICAL_BOOLEAN), + Model.ModelLimits(max_statistics_value_bytes=0)) + + unsigned = Model.LeafSpec(Model.PHYSICAL_INT32; + logical=Model.LOGICAL_UNSIGNED_INTEGER, bit_width=32) + @test Model.decode_bound(fill(0xff, 4), unsigned, + Model.ModelLimits()).value == Model.UnsignedValue(typemax(UInt32)) + signed8 = Model.LeafSpec(Model.PHYSICAL_INT32; + logical=Model.LOGICAL_SIGNED_INTEGER, bit_width=8) + @test Model.decode_bound(UInt8[0x7f, 0x00, 0x00, 0x00], signed8, + Model.ModelLimits()).state == Model.BOUND_KNOWN + @test Model.decode_bound(UInt8[0x80, 0x00, 0x00, 0x00], signed8, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + + uuid = Model.LeafSpec(Model.PHYSICAL_FIXED_LEN_BYTE_ARRAY; + logical=Model.LOGICAL_UUID, type_length=16) + @test_throws Model.ModelFormatError Model.decode_bound(fill(0x00, 15), uuid, + Model.ModelLimits(max_statistics_value_bytes=1)) + millis = Model.LeafSpec(Model.PHYSICAL_INT32; + logical=Model.LOGICAL_TIME, time_unit=Model.TIME_MILLIS) + @test Model.decode_bound(UInt8[0xff, 0x5b, 0x26, 0x05], millis, + Model.ModelLimits()).state == Model.BOUND_KNOWN + @test Model.decode_bound(UInt8[0x00, 0x5c, 0x26, 0x05], millis, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + + stringleaf = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY; + logical=Model.LOGICAL_STRING) + over = Model.decode_bound(UInt8[0xff, 0xff], stringleaf, + Model.ModelLimits(max_statistics_value_bytes=1)) + @test (over.state, over.reason, over.semantic_checked) == + (Model.BOUND_UNKNOWN, :over_limit, false) + invalid = Model.decode_bound(UInt8[0xff], stringleaf, + Model.ModelLimits(max_statistics_value_bytes=1)) + @test (invalid.state, invalid.reason, invalid.semantic_checked) == + (Model.BOUND_UNKNOWN, :invalid_logical, true) + + jsonleaf = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY; + logical=Model.LOGICAL_JSON) + validjson = Vector{UInt8}(codeunits("{\"a\":[1,true,null]}")) + @test Model.decode_bound(validjson, jsonleaf, + Model.ModelLimits()).state == Model.BOUND_KNOWN + @test Model.decode_bound(Vector{UInt8}(codeunits("{\"a\":}")), jsonleaf, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + + bsonleaf = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY; + logical=Model.LOGICAL_BSON) + validbson = UInt8[0x0c, 0x00, 0x00, 0x00, 0x10, 0x78, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00] + @test Model.decode_bound(validbson, bsonleaf, + Model.ModelLimits()).state == Model.BOUND_KNOWN + invalidbson = copy(validbson) + invalidbson[1] = 0x0d + @test Model.decode_bound(invalidbson, bsonleaf, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + validarray = UInt8[0x11, 0x00, 0x00, 0x00, 0x04, 0x61, 0x00, + 0x09, 0x00, 0x00, 0x00, 0x08, 0x30, 0x00, 0x01, 0x00, 0x00] + @test Model.decode_bound(validarray, bsonleaf, + Model.ModelLimits()).state == Model.BOUND_KNOWN + invalidarray = copy(validarray) + invalidarray[14] = 0x78 + @test Model.decode_bound(invalidarray, bsonleaf, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + validoldbinary = UInt8[0x14, 0x00, 0x00, 0x00, 0x05, 0x62, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x02, 0x03, 0x00, 0x00, 0x00, + 0x61, 0x62, 0x63, 0x00] + @test Model.decode_bound(validoldbinary, bsonleaf, + Model.ModelLimits()).state == Model.BOUND_KNOWN + invalidoldbinary = copy(validoldbinary) + invalidoldbinary[13] = 0x04 + @test Model.decode_bound(invalidoldbinary, bsonleaf, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + for subtype in UInt8[0x00, 0x09, 0x80] + @test Model.decode_bound(bsonbinary(subtype), bsonleaf, + Model.ModelLimits()).state == Model.BOUND_KNOWN + end + for subtype in UInt8[0x0a, 0x7f] + @test Model.decode_bound(bsonbinary(subtype), bsonleaf, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + end + @test Model.decode_bound(bsonregex("imsux"), bsonleaf, + Model.ModelLimits()).state == Model.BOUND_KNOWN + for options in ("mi", "ii", "z") + @test Model.decode_bound(bsonregex(options), bsonleaf, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + end + end + + @testset "decimal and byte comparisons" begin + decimal = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY; + logical=Model.LOGICAL_DECIMAL, precision=3) + minus129 = Model.decode_bound(UInt8[0xff, 0x7f], decimal, + Model.ModelLimits()).value + minus1 = Model.decode_bound(UInt8[0xff], decimal, + Model.ModelLimits()).value + plus1wide = Model.decode_bound(UInt8[0x00, 0x01], decimal, + Model.ModelLimits()).value + plus1 = Model.decode_bound(UInt8[0x01], decimal, + Model.ModelLimits()).value + @test Model.compare_values(minus129, minus1, Model.COMPARATOR_DECIMAL) == -1 + @test Model.compare_values(plus1wide, plus1, Model.COMPARATOR_DECIMAL) == 0 + + precision2 = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY; + logical=Model.LOGICAL_DECIMAL, precision=2) + @test Model.decode_bound(UInt8[0x63], precision2, + Model.ModelLimits()).state == Model.BOUND_KNOWN + @test Model.decode_bound(UInt8[0x64], precision2, + Model.ModelLimits()).state == Model.BOUND_UNKNOWN + + rawleaf = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY) + left = Model.decode_bound(UInt8[0x61, 0x00], rawleaf, + Model.ModelLimits()).value + right = Model.decode_bound(UInt8[0x61, 0x01], rawleaf, + Model.ModelLimits()).value + @test Model.compare_values(left, right, Model.COMPARATOR_UNSIGNED_BYTES) == -1 + @test Model.decode_bound(UInt8[0xff], rawleaf, + Model.ModelLimits()).state == Model.BOUND_KNOWN + + decimal16 = Model.LeafSpec(Model.PHYSICAL_FIXED_LEN_BYTE_ARRAY; + logical=Model.LOGICAL_DECIMAL, type_length=2, precision=5) + limits = Model.ModelLimits() + @test all(begin + leftvalue = Model.decode_bound(be16(Int16(value)), decimal16, + limits).value + rightvalue = Model.decode_bound(be16(Int16(value + 1)), decimal16, + limits).value + Model.compare_values(leftvalue, rightvalue, + Model.COMPARATOR_DECIMAL) == -1 + end for value in Int32(typemin(Int16)):(Int32(typemax(Int16)) - 1)) + end + + @testset "raw-bit IEEE total order" begin + patterns = UInt16[0xfe01, 0xfc01, 0xfc00, 0x8000, 0x0000, 0x7c00, + 0x7c01, 0x7e01] + values = [Model.FloatValue(UInt8(16), UInt64(bits)) for bits in patterns] + @test all(Model.compare_values(values[index], values[index + 1], + Model.COMPARATOR_IEEE_FLOAT) == -1 for index in 1:(length(values) - 1)) + @test Model.compare_values(values[4], values[5], + Model.COMPARATOR_TYPE_FLOAT) == 0 + @test length(unique(Model.ieee_total_key(value) for value in values)) == + length(values) + samples = ( + (UInt8(32), UInt64[0xffffffff, 0xff800000, 0x80000001, + 0x80000000, 0x00000000, 0x00000001, 0x7f800000, + 0x7fffffff]), + (UInt8(64), UInt64[0xffffffffffffffff, 0xfff0000000000000, + 0x8000000000000001, 0x8000000000000000, + 0x0000000000000000, 0x0000000000000001, + 0x7ff0000000000000, 0x7fffffffffffffff]), + ) + for (width, bits) in samples + ordered = [Model.FloatValue(width, value) for value in bits] + @test all(Model.compare_values(ordered[index], ordered[index + 1], + Model.COMPARATOR_IEEE_FLOAT) == -1 for + index in 1:(length(ordered) - 1)) + @test Model.float_isnan(first(ordered)) + @test Model.float_isnan(last(ordered)) + end + end + + @testset "statistics value limit" begin + rawleaf = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY) + exact = Model.decode_bound(UInt8[1, 2, 3, 4], rawleaf, + Model.ModelLimits(max_statistics_value_bytes=4)) + @test exact.state == Model.BOUND_KNOWN + @test Model.decode_bound(UInt8[1, 2, 3, 4, 5], rawleaf, + Model.ModelLimits(max_statistics_value_bytes=4)).state == + Model.BOUND_UNKNOWN + @test Model.writer_bounds_allowed(UInt8[1, 2, 3, 4], UInt8[5], + Model.ModelLimits(max_statistics_value_bytes=4)) + @test !Model.writer_bounds_allowed(UInt8[1, 2, 3, 4, 5], UInt8[5], + Model.ModelLimits(max_statistics_value_bytes=4)) + zerolimits = Model.ModelLimits(max_statistics_value_bytes=0) + @test Model.decode_bound(UInt8[], rawleaf, zerolimits).state == + Model.BOUND_KNOWN + overzero = Model.decode_bound(UInt8[0xff], rawleaf, zerolimits) + @test (overzero.state, overzero.reason, overzero.semantic_checked) == + (Model.BOUND_UNKNOWN, :over_limit, false) + @test Model.writer_bounds_allowed(UInt8[], UInt8[], zerolimits) + @test !Model.writer_bounds_allowed(UInt8[0x00], UInt8[], zerolimits) + @test_throws ArgumentError Model.writer_bounds_allowed(UInt8[], UInt8[], + Model.ModelLimits(max_statistics_value_bytes=-1)) + @test_throws ArgumentError Model.producer_decision( + "parquet-cpp version 1.2.9", rawleaf, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, + UInt8[], UInt8[], Model.ModelLimits(max_statistics_value_bytes=-1)) + for createdby in ("parquet-cpp version 1.2.9", + "parquet-mr version 1.9.9"), equal in (false, true) + lower = fill(UInt8(0x61), 5) + upper = equal ? copy(lower) : fill(UInt8(0x62), 5) + result = Model.interpret_statistics(rawleaf, Int64(1), + Model.RawStatistics(modern_lower=lower, modern_upper=upper), + Model.DeclaredOrder[Model.ORDER_TYPE]; created_by=createdby, + limits=Model.ModelLimits(max_statistics_value_bytes=4)) + @test result.trust.state == Model.TRUST_UNTRUSTED + @test result.trust.reason == :legacy_wrong_order + @test result.lower.reason == :over_limit + @test result.upper.reason == :over_limit + end + malformed = Model.RawStatistics(modern_lower=UInt8[0x00], + null_count=Int64(-1)) + @test_throws ArgumentError Model.interpret_statistics(rawleaf, Int64(-1), + malformed, Model.DeclaredOrder[]; + limits=Model.ModelLimits(max_statistics_value_bytes=-1)) + end + + @testset "count state machine" begin + floatleaf = Model.LeafSpec(Model.PHYSICAL_FLOAT) + stats = Model.RawStatistics(null_count=Int64(2), nan_count=Int64(3), + distinct_count=Int64(4)) + result = Model.interpret_statistics(floatleaf, Int64(10), stats, + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test result.occupancy == Model.OCCUPANCY_HAS_NON_NAN + @test (result.null_count.known, result.null_count.value) == (true, 2) + @test (result.nan_count.known, result.nan_count.value) == (true, 3) + @test (result.distinct_count.known, result.distinct_count.value) == (true, 4) + + @test_throws Model.ModelFormatError Model.interpret_statistics(floatleaf, + Int64(4), Model.RawStatistics(null_count=Int64(2), nan_count=Int64(3)), + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test_throws Model.ModelFormatError Model.interpret_statistics( + Model.LeafSpec(Model.PHYSICAL_INT32), Int64(4), + Model.RawStatistics(nan_count=Int64(0)), + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test_throws Model.ModelFormatError Model.interpret_statistics(floatleaf, + Int64(4), Model.RawStatistics(null_count=Int64(2), + distinct_count=Int64(3)), Model.DeclaredOrder[Model.ORDER_TYPE]) + @test_throws Model.ModelFormatError Model.interpret_statistics(floatleaf, + Int64(4), Model.RawStatistics(null_count=Int64(-1)), + Model.DeclaredOrder[Model.ORDER_TYPE]) + + zero = Model.interpret_statistics(floatleaf, Int64(0), + Model.RawStatistics(null_count=Int64(0), nan_count=Int64(0), + distinct_count=Int64(0)), + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test zero.occupancy == Model.OCCUPANCY_EMPTY + @test all(fact -> fact.known && iszero(fact.value), + (zero.null_count, zero.nan_count, zero.distinct_count)) + missing = Model.interpret_statistics(floatleaf, Int64(0), + Model.RawStatistics(), Model.DeclaredOrder[Model.ORDER_TYPE]) + @test all(fact -> !fact.known, + (missing.null_count, missing.nan_count, missing.distinct_count)) + @test missing.occupancy == Model.OCCUPANCY_EMPTY + + intleaf = Model.LeafSpec(Model.PHYSICAL_INT32) + emptyfamilies = ( + Model.RawStatistics(modern_lower=le32(UInt32(1)), + modern_upper=le32(UInt32(2))), + Model.RawStatistics(deprecated_lower=le32(UInt32(1)), + deprecated_upper=le32(UInt32(2))), + ) + for emptystats in emptyfamilies + emptyresult = Model.interpret_statistics(intleaf, Int64(0), + emptystats, Model.DeclaredOrder[Model.ORDER_TYPE]) + @test emptyresult.occupancy == Model.OCCUPANCY_EMPTY + @test (emptyresult.lower.state, emptyresult.lower.reason) == + (Model.BOUND_UNKNOWN, :no_non_null_values) + @test (emptyresult.upper.state, emptyresult.upper.reason) == + (Model.BOUND_UNKNOWN, :no_non_null_values) + end + largest = Model.interpret_statistics(floatleaf, typemax(Int64), + Model.RawStatistics(null_count=typemax(Int64), nan_count=Int64(0)), + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test largest.occupancy == Model.OCCUPANCY_EMPTY + end + + @testset "atomic bound-family selection" begin + intleaf = Model.LeafSpec(Model.PHYSICAL_INT32) + modernlower = le32(UInt32(1)) + deprecatedupper = le32(UInt32(9)) + stats = Model.RawStatistics(modern_lower=modernlower, + deprecated_upper=deprecatedupper) + result = Model.interpret_statistics(intleaf, Int64(1), stats, + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test result.family == Model.FAMILY_MODERN + @test result.lower.state == Model.BOUND_KNOWN + @test result.upper.state == Model.BOUND_ABSENT + + deprecated = Model.RawStatistics(deprecated_lower=le32(UInt32(1)), + deprecated_upper=le32(UInt32(9)), lower_exact=true, + upper_exact=true) + result = Model.interpret_statistics(intleaf, Int64(1), deprecated, nothing) + @test result.family == Model.FAMILY_DEPRECATED + @test result.lower.state == Model.BOUND_KNOWN + @test result.upper.state == Model.BOUND_KNOWN + @test result.lower.exactness == Model.EXACTNESS_UNKNOWN + @test result.upper.exactness == Model.EXACTNESS_UNKNOWN + + unsigned = Model.LeafSpec(Model.PHYSICAL_INT32; + logical=Model.LOGICAL_UNSIGNED_INTEGER, bit_width=32) + result = Model.interpret_statistics(unsigned, Int64(1), deprecated, nothing) + @test result.lower.state == Model.BOUND_UNKNOWN + @test result.upper.state == Model.BOUND_UNKNOWN + + @test_throws Model.ModelFormatError Model.interpret_statistics(intleaf, + Int64(1), stats, Model.DeclaredOrder[]) + @test_throws Model.ModelFormatError Model.interpret_statistics(intleaf, + Int64(1), stats, + Model.DeclaredOrder[Model.ORDER_TYPE, Model.ORDER_TYPE]) + missingorder = Model.interpret_statistics(intleaf, Int64(1), stats, nothing) + @test missingorder.lower.reason == :missing_column_orders + future = Model.interpret_statistics(intleaf, Int64(1), stats, + Model.DeclaredOrder[Model.ORDER_FUTURE]) + @test future.lower.reason == :unknown_column_order + @test_throws Model.ModelFormatError Model.interpret_statistics(intleaf, + Int64(1), stats, Model.DeclaredOrder[Model.ORDER_IEEE]) + + contradictory = Model.RawStatistics(modern_lower=le32(UInt32(9)), + modern_upper=le32(UInt32(1))) + result = Model.interpret_statistics(intleaf, Int64(1), contradictory, + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test result.lower.reason == :contradictory_bounds + @test result.upper.reason == :contradictory_bounds + + int96 = Model.LeafSpec(Model.PHYSICAL_INT96) + undefined = Model.interpret_statistics(int96, Int64(1), + Model.RawStatistics(modern_lower=fill(0x00, 12)), + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test undefined.comparator == Model.COMPARATOR_UNDEFINED + @test undefined.lower.reason == :undefined_type_order + + binary = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY) + emptystats = Model.RawStatistics(modern_lower=UInt8[0x61], + modern_upper=UInt8[0x62], null_count=Int64(4)) + for createdby in ("parquet-cpp version 1.2.9", + "parquet-mr version 1.9.9"), modelorders in ( + nothing, Model.DeclaredOrder[Model.ORDER_FUTURE]) + result = Model.interpret_statistics(binary, Int64(4), emptystats, + modelorders; created_by=createdby) + expected = modelorders === nothing ? :missing_column_orders : + :unknown_column_order + @test result.occupancy == Model.OCCUPANCY_EMPTY + @test result.trust.state == Model.TRUST_UNTRUSTED + @test result.trust.reason == :legacy_wrong_order + @test result.lower.reason == expected + @test result.upper.reason == expected + end + + limited = Model.interpret_statistics( + Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY), Int64(1), + Model.RawStatistics(modern_lower=UInt8[0x61, 0x62], + modern_upper=UInt8[0x7a], lower_exact=true, + upper_exact=false), Model.DeclaredOrder[Model.ORDER_TYPE]; + created_by="impala version 1.0.0", + limits=Model.ModelLimits(max_statistics_value_bytes=1)) + @test (limited.lower.state, limited.lower.reason, + limited.lower.exactness) == + (Model.BOUND_UNKNOWN, :over_limit, Model.EXACTNESS_EXACT) + @test (limited.upper.state, limited.upper.exactness) == + (Model.BOUND_KNOWN, Model.EXACTNESS_INEXACT) + end + + @testset "TYPE_ORDER floating compatibility" begin + floatleaf = Model.LeafSpec(Model.PHYSICAL_FLOAT) + zeros = Model.RawStatistics(modern_lower=le32(UInt32(0x00000000)), + modern_upper=le32(UInt32(0x80000000)), + lower_exact=true, upper_exact=true, null_count=Int64(0), + nan_count=Int64(0)) + result = Model.interpret_statistics(floatleaf, Int64(2), zeros, + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test result.lower.value == Model.FloatValue(UInt8(32), UInt64(0x80000000)) + @test result.upper.value == Model.FloatValue(UInt8(32), UInt64(0x00000000)) + @test result.lower.exactness == Model.EXACTNESS_INEXACT + @test result.upper.exactness == Model.EXACTNESS_INEXACT + + nan = le32(UInt32(0x7fc00001)) + finite = le32(UInt32(0x3f800000)) + partial = Model.RawStatistics(modern_lower=nan, modern_upper=finite) + result = Model.interpret_statistics(floatleaf, Int64(2), partial, + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test result.lower.reason == :nan_type_order + @test result.upper.state == Model.BOUND_KNOWN + + allnan = Model.RawStatistics(modern_lower=nan, modern_upper=nan, + null_count=Int64(0), nan_count=Int64(2)) + result = Model.interpret_statistics(floatleaf, Int64(2), allnan, + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test result.lower.reason == :all_nan_type_order + @test result.upper.reason == :all_nan_type_order + + allnull = Model.RawStatistics(modern_lower=finite, modern_upper=finite, + null_count=Int64(2), nan_count=Int64(0)) + result = Model.interpret_statistics(floatleaf, Int64(2), allnull, + Model.DeclaredOrder[Model.ORDER_TYPE]) + @test result.lower.reason == :no_non_null_values + @test result.upper.reason == :no_non_null_values + end + + @testset "IEEE count-bound states" begin + floatleaf = Model.LeafSpec(Model.PHYSICAL_FLOAT) + signaling = le32(UInt32(0x7f800001)) + quiet = le32(UInt32(0x7fc00001)) + finite = le32(UInt32(0x3f800000)) + allnan = Model.RawStatistics(modern_lower=signaling, + modern_upper=quiet, null_count=Int64(0), nan_count=Int64(2)) + result = Model.interpret_statistics(floatleaf, Int64(2), allnan, + Model.DeclaredOrder[Model.ORDER_IEEE]) + @test result.lower.state == Model.BOUND_KNOWN + @test result.upper.state == Model.BOUND_KNOWN + @test result.comparator == Model.COMPARATOR_IEEE_FLOAT + + badallnan = Model.RawStatistics(modern_lower=finite, + modern_upper=quiet, null_count=Int64(0), nan_count=Int64(2)) + result = Model.interpret_statistics(floatleaf, Int64(2), badallnan, + Model.DeclaredOrder[Model.ORDER_IEEE]) + @test result.lower.reason == :ieee_bound_kind_contradiction + @test result.upper.reason == :ieee_bound_kind_contradiction + + mixed = Model.RawStatistics(modern_lower=finite, + modern_upper=le32(UInt32(0x40000000)), null_count=Int64(0), + nan_count=Int64(1)) + result = Model.interpret_statistics(floatleaf, Int64(3), mixed, + Model.DeclaredOrder[Model.ORDER_IEEE]) + @test result.lower.state == Model.BOUND_KNOWN + @test result.upper.state == Model.BOUND_KNOWN + + badmixed = Model.RawStatistics(modern_lower=finite, modern_upper=quiet, + null_count=Int64(0), nan_count=Int64(1)) + result = Model.interpret_statistics(floatleaf, Int64(3), badmixed, + Model.DeclaredOrder[Model.ORDER_IEEE]) + @test result.lower.reason == :ieee_bound_kind_contradiction + @test result.upper.reason == :ieee_bound_kind_contradiction + + unknown = Model.RawStatistics(modern_lower=quiet, modern_upper=finite) + result = Model.interpret_statistics(floatleaf, Int64(3), unknown, + Model.DeclaredOrder[Model.ORDER_IEEE]) + @test result.lower.reason == :unproven_ieee_nan + @test result.upper.state == Model.BOUND_KNOWN + + onefinite = Model.RawStatistics(modern_lower=finite, + null_count=Int64(0), nan_count=Int64(2)) + result = Model.interpret_statistics(floatleaf, Int64(2), onefinite, + Model.DeclaredOrder[Model.ORDER_IEEE]) + @test result.lower.reason == :ieee_bound_kind_contradiction + @test result.upper.state == Model.BOUND_ABSENT + end + + @testset "producer-version policy" begin + binary = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY) + lower = UInt8[0x61] + upper = UInt8[0x62] + for application in ("parquet-cpp", "parquet-mr", "impala") + @test !Model.parse_created_by(application).parsed + end + for application in ("parquet-cpp", "parquet-mr") + arrow = Model.parse_arrow_created_by(application) + @test arrow.parsed + @test arrow.application == application + @test arrow.version === nothing + end + @test !Model.parse_arrow_created_by("impala").parsed + @test Model.parse_created_by( + "parquet-mr version 1.8.0").parsed + backtracked = Model.parse_created_by( + "foo version (bad) bar version 1.0.0") + @test backtracked.parsed + @test backtracked.application == "foo version (bad) bar" + for created_by in ("parquet-mr version (bad) x version 1.7.9", + "parquet-cpp version (bad) x version 1.2.9") + producer = Model.parse_created_by(created_by) + @test producer.parsed + @test producer.application in ( + "parquet-mr version (bad) x", "parquet-cpp version (bad) x") + @test Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, + lower, upper).state == Model.TRUST_TRUSTED + end + p251affected = Union{Nothing,String}[nothing, "", " version 1.0.0", + "parquet-mr", "parquet-mr version 1", "parquet-mr version 1.2", + "parquet-mr version 1.7.9", + "parquet-mr version 1.8.0-rc1", + "parquet-mr version 1.5.0-cdh5.4.9", + "parquet-mr version 1.5.0-.", + "parquet-mr version 1.5.0-..", + "parquet-mr version 1.5.0-cdh5.5.", + "parquet-mr version 1.5.0-cdh5.5.."] + for created_by in p251affected + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, lower, upper) + @test decision.state == Model.TRUST_UNTRUSTED + @test decision.reason == :parquet_251 + end + for created_by in ("impala", "garbage!") + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, + lower, upper) + @test decision.state == Model.TRUST_UNTRUSTED + @test decision.reason == :parquet_251 + end + @test Model.producer_decision("parquet-cpp", binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, + lower, lower).reason == :parquet_251 + for created_by in ("parquet-mr version 1.8.0", + "parquet-mr version 1.5.0-cdh5.5.0") + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, lower, upper) + @test decision.state == Model.TRUST_UNTRUSTED + @test decision.reason == :legacy_wrong_order + end + for (created_by, expected) in ( + ("parquet-mr version 1.5.0-cdh5.5.0", Model.TRUST_TRUSTED), + ("parquet-mr version 1.5.0-cdh5.5.0.", Model.TRUST_TRUSTED), + ("parquet-mr version 1.5.0-cdh5.5.0..", Model.TRUST_TRUSTED), + ("parquet-mr version 1.5.0-cdh5.5.0+build.7", + Model.TRUST_TRUSTED), + ("parquet-mr\tversion\t1.8.0", Model.TRUST_TRUSTED), + ("parquet-mr version 1.5.0-cdh5.4.9", Model.TRUST_UNTRUSTED), + ("parquet-mr version 1.5.0", Model.TRUST_UNTRUSTED), + ("parquet-mr version 1.5.0zz-cdh5.5.0", + Model.TRUST_UNTRUSTED), + ("parquet-mr version 1.5.0-cdh5.2147483648x.0", + Model.TRUST_TRUSTED), + ("parquet-mr version 1.5.0-cdh5.-1.0", + Model.TRUST_TRUSTED), + ("parquet-mr version 1.5.0-cdh5. 5.0", + Model.TRUST_TRUSTED), + ("parquet-mr version 1.5.0-cdh5.\u0665.0", + Model.TRUST_TRUSTED), + ("parquet-mr version 2147483648.0.0", + Model.TRUST_UNTRUSTED)) + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_SIGNED, Model.FAMILY_MODERN, lower, upper) + @test decision.state == expected + end + @test Model.producer_decision("impala version 1.0.0", binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, lower, + upper).state == Model.TRUST_TRUSTED + for created_by in ("parquet-cpp version 1.3.1-2147483648x", + "parquet-cpp version 1.3.1-2147483648)", + "parquet-mr version 1.10.1-2147483648x", + "parquet-mr version 1.10.1-2147483648)") + @test Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, + lower, upper).state == Model.TRUST_TRUSTED + end + for terminator in ("\n", "\r", "\r\n", "\u0085", "\u2028", "\u2029") + for (application, version, reason) in ( + ("parquet-mr", "1.10.0", :parquet_251), + ("parquet-cpp", "1.3.0", :legacy_wrong_order)) + created_by = application * " version " * version * "+foo" * + terminator * "bar" + @test Model.parse_created_by(created_by).version === nothing + @test Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, + lower, upper).reason == reason + end + end + for separator in ("\v", "\f") + for created_by in ("parquet-mr version 1.10.0+foo" * separator * "bar", + "parquet-cpp version 1.3.0+foo" * separator * "bar") + @test Model.parse_created_by(created_by).version !== nothing + @test Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, + lower, upper).state == Model.TRUST_TRUSTED + end + end + equal = UInt8[0x61] + @test Model.producer_decision("parquet-mr version 1.7.9", binary, + Model.COMPARATOR_UNSIGNED_BYTES, Model.FAMILY_MODERN, equal, + equal).reason == :parquet_251 + + for terminator in ("\n", "\r", "\r\n", "\u0085", "\u2028", "\u2029") + created_by = "foo" * terminator * "bar version 1.0.0" + @test !Model.parse_created_by(created_by).parsed + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_SIGNED, Model.FAMILY_MODERN, lower, upper) + @test decision.state == Model.TRUST_UNTRUSTED + @test decision.reason == :parquet_251 + end + for separator in ("\n", "\r", "\r\n", "\v", "\f") + created_by = "parquet-mr" * separator * "version 1.8.0" + parsed = Model.parse_created_by(created_by) + @test parsed.parsed + @test parsed.application == "parquet-mr" + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_SIGNED, Model.FAMILY_MODERN, lower, upper) + @test decision.state == Model.TRUST_TRUSTED + @test decision.reason == :trusted + end + for separator in ("\u0085", "\u2028", "\u2029") + created_by = "parquet-mr" * separator * "version 1.8.0" + @test !Model.parse_created_by(created_by).parsed + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_SIGNED, Model.FAMILY_MODERN, lower, upper) + @test decision.state == Model.TRUST_UNTRUSTED + @test decision.reason == :parquet_251 + end + + oldcases = [ + ("parquet-cpp", true), + ("parquet-cpp version 1", true), + ("parquet-cpp version 1.2", true), + ("parquet-cpp version 1.2.9", true), + ("parquet-cpp version 1.3", true), + ("parquet-cpp version", true), + ("parquet-cpp version1.2.9", true), + ("parquet-cpp version 1.3.0-rc1", true), + ("parquet-cpp version 1.3.0", false), + ("parquet-mr", true), + ("parquet-mr version 1", true), + ("parquet-mr version 1.2", true), + ("parquet-mr version 1.9.9", true), + ("parquet-mr version 1.10", true), + ("parquet-mr version 1.10.0-rc1", true), + ("parquet-mr version 1.10.0", false), + ] + intleaf = Model.LeafSpec(Model.PHYSICAL_INT32; + logical=Model.LOGICAL_UNSIGNED_INTEGER, bit_width=32) + for (created_by, affected) in oldcases + decision = Model.producer_decision(created_by, intleaf, + Model.COMPARATOR_UNSIGNED, Model.FAMILY_MODERN, le32(UInt32(1)), + le32(UInt32(2))) + @test (decision.state == Model.TRUST_UNTRUSTED) == affected + end + @test Model.producer_decision("parquet-cpp version 1.2.9", intleaf, + Model.COMPARATOR_UNSIGNED, Model.FAMILY_DEPRECATED, equal, + equal).state == Model.TRUST_TRUSTED + floatleaf = Model.LeafSpec(Model.PHYSICAL_FLOAT) + @test Model.producer_decision("parquet-mr version 1.9.9", floatleaf, + Model.COMPARATOR_IEEE_FLOAT, Model.FAMILY_MODERN, + le32(UInt32(0)), le32(UInt32(1))).state == Model.TRUST_UNTRUSTED + @test Model.producer_decision("parquet-mr version 1.9.9", floatleaf, + Model.COMPARATOR_TYPE_FLOAT, Model.FAMILY_MODERN, + le32(UInt32(0)), le32(UInt32(1))).state == Model.TRUST_TRUSTED + + rejected = Model.interpret_statistics(binary, Int64(3), + Model.RawStatistics(modern_lower=lower, modern_upper=upper, + null_count=Int64(1), distinct_count=Int64(2)), + Model.DeclaredOrder[Model.ORDER_TYPE]; + created_by="parquet-mr version 1.7.9") + @test rejected.lower.reason == :parquet_251 + @test rejected.upper.reason == :parquet_251 + @test (rejected.null_count.known, rejected.null_count.value) == (true, 1) + @test (rejected.distinct_count.known, rejected.distinct_count.value) == + (true, 2) + end + + @testset "exhaustive Float16 total order" begin + patterns = collect(UInt16(0):typemax(UInt16)) + sort!(patterns; lt=(left, right) -> Model.compare_values( + Model.FloatValue(UInt8(16), UInt64(left)), + Model.FloatValue(UInt8(16), UInt64(right)), + Model.COMPARATOR_IEEE_FLOAT) < 0) + @test length(patterns) == 65_536 + @test first(patterns) == UInt16(0xffff) + @test last(patterns) == UInt16(0x7fff) + @test all(Model.compare_values( + Model.FloatValue(UInt8(16), UInt64(patterns[index])), + Model.FloatValue(UInt8(16), UInt64(patterns[index + 1])), + Model.COMPARATOR_IEEE_FLOAT) == -1 for index in 1:65_535) + keys = [Model.ieee_total_key(Model.FloatValue(UInt8(16), UInt64(bits))) + for bits in patterns] + @test length(unique(keys)) == 65_536 + @test count(bits -> Model.float_isnan( + Model.FloatValue(UInt8(16), UInt64(bits))), patterns) == 2_046 + orderedbytes = Vector{UInt8}(undef, 2 * length(patterns)) + for (index, bits) in enumerate(patterns) + orderedbytes[2 * index - 1] = UInt8(bits & 0xff) + orderedbytes[2 * index] = UInt8(bits >> 8) + end + @test bytes2hex(sha256(orderedbytes)) == + "61619b1a4260ee4cff6d462d21cab5a049198a27b3a9ae3e8ecfe246efc1d4b6" + end + + @testset "independent extrema summary" begin + float16 = Model.LeafSpec(Model.PHYSICAL_FIXED_LEN_BYTE_ARRAY; + logical=Model.LOGICAL_FLOAT16, type_length=2) + mixed = Vector{UInt8}[ + le16(UInt16(0x7e01)), + le16(UInt16(0xbc00)), + le16(UInt16(0x4000)), + le16(UInt16(0x7c01)), + ] + summary = Model.summarize_raw_values(float16, mixed, + Model.COMPARATOR_IEEE_FLOAT) + @test summary.nan_count == 2 + @test summary.lower == Model.FloatValue(UInt8(16), UInt64(0xbc00)) + @test summary.upper == Model.FloatValue(UInt8(16), UInt64(0x4000)) + @test Model.contains_value(summary, + Model.FloatValue(UInt8(16), UInt64(0x3c00)), + Model.COMPARATOR_IEEE_FLOAT) + + allnan = Vector{UInt8}[ + le16(UInt16(0x7e02)), + le16(UInt16(0x7c01)), + le16(UInt16(0xfe03)), + ] + summary = Model.summarize_raw_values(float16, allnan, + Model.COMPARATOR_IEEE_FLOAT) + @test summary.nan_count == 3 + @test summary.lower == Model.FloatValue(UInt8(16), UInt64(0xfe03)) + @test summary.upper == Model.FloatValue(UInt8(16), UInt64(0x7e02)) + @test all(value -> Model.contains_value(summary, value, + Model.COMPARATOR_IEEE_FLOAT), ( + Model.FloatValue(UInt8(16), UInt64(0xfe03)), + Model.FloatValue(UInt8(16), UInt64(0x7c01)), + Model.FloatValue(UInt8(16), UInt64(0x7e02)))) + @test !Model.contains_value(summary, + Model.FloatValue(UInt8(16), UInt64(0xffff)), + Model.COMPARATOR_IEEE_FLOAT) + @test !Model.contains_value(Model.summarize_raw_values(float16, mixed, + Model.COMPARATOR_IEEE_FLOAT), + Model.FloatValue(UInt8(16), UInt64(0x7e01)), + Model.COMPARATOR_IEEE_FLOAT) + + typesummary = Model.summarize_raw_values(float16, mixed, + Model.COMPARATOR_TYPE_FLOAT) + @test typesummary.nan_count == 2 + @test typesummary.lower == Model.FloatValue(UInt8(16), UInt64(0xbc00)) + @test typesummary.upper == Model.FloatValue(UInt8(16), UInt64(0x4000)) + typeallnan = Model.summarize_raw_values(float16, allnan, + Model.COMPARATOR_TYPE_FLOAT) + @test typeallnan.lower === nothing + @test typeallnan.upper === nothing + + pluszero = le16(UInt16(0x0000)) + minuszero = le16(UInt16(0x8000)) + plussummary = Model.summarize_raw_values(float16, + Vector{UInt8}[pluszero], Model.COMPARATOR_IEEE_FLOAT) + @test plussummary.lower == plussummary.upper == + Model.FloatValue(UInt8(16), UInt64(0x0000)) + minussummary = Model.summarize_raw_values(float16, + Vector{UInt8}[minuszero], Model.COMPARATOR_IEEE_FLOAT) + @test minussummary.lower == minussummary.upper == + Model.FloatValue(UInt8(16), UInt64(0x8000)) + bothsummary = Model.summarize_raw_values(float16, + Vector{UInt8}[pluszero, minuszero], Model.COMPARATOR_IEEE_FLOAT) + @test bothsummary.lower == Model.FloatValue(UInt8(16), UInt64(0x8000)) + @test bothsummary.upper == Model.FloatValue(UInt8(16), UInt64(0x0000)) + + rawvalues = Vector{UInt8}[UInt8[0x61], UInt8[0x7a]] + bytesummary = Model.summarize_raw_values( + Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY), rawvalues, + Model.COMPARATOR_UNSIGNED_BYTES) + rawvalues[1][1] = 0xff + rawvalues[2][1] = 0x00 + @test bytesummary.lower isa Model.ByteValue + @test bytesummary.upper isa Model.ByteValue + @test bytesummary.lower.value == UInt8[0x61] + @test bytesummary.upper.value == UInt8[0x7a] + @test bytesummary.lower.value !== rawvalues[1] + @test bytesummary.upper.value !== rawvalues[2] + end + + @testset "producer policy cross-product" begin + unsigned = Model.LeafSpec(Model.PHYSICAL_INT32; + logical=Model.LOGICAL_UNSIGNED_INTEGER, bit_width=32) + families = (Model.FAMILY_MODERN, Model.FAMILY_DEPRECATED) + comparators = (Model.COMPARATOR_SIGNED, Model.COMPARATOR_UNSIGNED, + Model.COMPARATOR_IEEE_FLOAT) + cutoffs = ( + ("parquet-cpp version 1.3.0-rc1", true), + ("parquet-cpp version 1.3.0", false), + ("parquet-mr version 1.10.0-rc1", true), + ("parquet-mr version 1.10.0", false), + ) + for (created_by, affected) in cutoffs, family in families, + comparator in comparators, equal in (false, true) + lower = le32(UInt32(1)) + upper = equal ? copy(lower) : le32(UInt32(2)) + decision = Model.producer_decision(created_by, unsigned, comparator, + family, lower, upper) + nonsigned = comparator != Model.COMPARATOR_SIGNED + expected = affected && nonsigned && !equal + @test (decision.state == Model.TRUST_UNTRUSTED) == expected + end + + binary = Model.LeafSpec(Model.PHYSICAL_BYTE_ARRAY) + versions = ( + ("parquet-mr version 1.7.9", :parquet_251), + ("parquet-mr version 1.8.0", :legacy_wrong_order), + ("parquet-mr version 1.5.0-cdh5.5.0", :legacy_wrong_order), + ("parquet-mr version 1.10.0", :trusted), + ) + for (created_by, distinct_reason) in versions, family in families, + equal in (false, true) + lower = UInt8[0x61] + upper = equal ? copy(lower) : UInt8[0x62] + decision = Model.producer_decision(created_by, binary, + Model.COMPARATOR_UNSIGNED_BYTES, family, lower, upper) + expected = equal && distinct_reason == :legacy_wrong_order ? :trusted : + distinct_reason + @test decision.reason == expected + end + end + + @testset "atomic family presence cross-product" begin + leaf = Model.LeafSpec(Model.PHYSICAL_INT32) + deprecatedlower = le32(UInt32(3)) + deprecatedupper = le32(UInt32(7)) + for haslower in (false, true), hasupper in (false, true) + stats = Model.RawStatistics( + modern_lower=haslower ? le32(UInt32(4)) : nothing, + modern_upper=hasupper ? le32(UInt32(6)) : nothing, + deprecated_lower=deprecatedlower, + deprecated_upper=deprecatedupper) + result = Model.interpret_statistics(leaf, Int64(1), stats, + Model.DeclaredOrder[Model.ORDER_TYPE]) + expectedfamily = haslower || hasupper ? Model.FAMILY_MODERN : + Model.FAMILY_DEPRECATED + @test result.family == expectedfamily + if expectedfamily == Model.FAMILY_MODERN + @test (result.lower.state == Model.BOUND_ABSENT) == !haslower + @test (result.upper.state == Model.BOUND_ABSENT) == !hasupper + else + @test result.lower.value == Model.SignedValue(Int128(3)) + @test result.upper.value == Model.SignedValue(Int128(7)) + end + end + end + + @testset "frozen manifest and forbidden dependencies" begin + manifest = TOML.parsefile(joinpath(@__DIR__, "cases.toml")) + @test manifest["schema_version"] == 1 + @test manifest["authority_plan_sha256"] == + "15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304" + @test manifest["parquet_format_commit"] == + "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" + @test manifest["arrow_statistics_policy_commit"] == + "515410b2a14ac766258e00b07eab9e5ee2692a62" + @test manifest["parquet_java_tag"] == "apache-parquet-1.17.1" + @test manifest["parquet_java_commit"] == + "78a8d3230eb4769db93de5f2f2e18363c04cae81" + @test manifest["parquet_java_embedded_format"] == "2.12" + @test manifest["module"] == "N6StatisticsModel.jl" + @test manifest["runner"] == "runtests.jl" + @test manifest["production_dependency_allowed"] == false + @test manifest["float16"]["pattern_count"] == 65_536 + @test manifest["float16"]["nan_pattern_count"] == 2_046 + @test manifest["float16"]["ordered_little_endian_sha256"] == + "61619b1a4260ee4cff6d462d21cab5a049198a27b3a9ae3e8ecfe246efc1d4b6" + @test manifest["limits"]["default_statistics_value_bytes"] == 4096 + @test manifest["limits"]["equality_succeeds"] == true + @test manifest["limits"]["fixed_structure_precedes_limit"] == true + @test manifest["limits"]["variable_limit_precedes_semantics"] == true + @test length(manifest["case_groups"]) == 13 + caseids = [group["id"] for group in manifest["case_groups"]] + @test length(unique(caseids)) == length(caseids) + for group in manifest["case_groups"] + @test Set(keys(group)) == Set(["id", "requirements", + "capabilities", "digest_contract", "expected_sha256"]) + @test group["id"] isa String + @test !isempty(group["requirements"]) + @test all(value -> value isa String, group["requirements"]) + @test group["capabilities"] == sort(group["capabilities"]) + @test length(group["capabilities"]) == + length(unique(group["capabilities"])) + @test group["digest_contract"] == + "n6-capability-result-sha256-v1" + @test Set(keys(group["expected_sha256"])) == + Set(group["capabilities"]) + @test all(value -> occursin(r"^[0-9a-f]{64}$", value), + values(group["expected_sha256"])) + end + + package_name = string("Par", "quet") + forbidden = String[ + string("using ", package_name), + string("import ", package_name), + string(package_name, "."), + string("include(\"", "..", "/"), + string("include(\"", "src", "/"), + string("src", "/", "statistics.jl"), + ] + juliafiles = sort(filter(path -> endswith(path, ".jl"), readdir(@__DIR__; + join=true))) + @test basename.(juliafiles) == ["N6StatisticsModel.jl", "runtests.jl"] + for path in juliafiles + source = read(path, String) + @test all(needle -> !occursin(needle, source), forbidden) + end + @test !isdefined(Main, Symbol(package_name)) + end +end diff --git a/test/conformance/n6/normalizers/README.md b/test/conformance/n6/normalizers/README.md new file mode 100644 index 0000000..6983d22 --- /dev/null +++ b/test/conformance/n6/normalizers/README.md @@ -0,0 +1,46 @@ +# N6 raw and semantic evidence normalization + +These tools transform the exact 20-record raw Java scanner output into the N6 +normalized schema. They then run the frozen independent Julia model over every +normalized column. They do not import or call Parquet.jl. + +The raw normalizer also supports the separate exact 12-record generated-fixture +scan. Use `--fixture-set generated` for evidence ID +`normalized-raw-java-generated`. This mode accepts only verified generated +fixture identities and the source-pinned raw input digest. It emits file and +column facts for all 12 cases and PASS results only for reviewed raw-wire claims. + +`normalize_raw.py` accepts only the raw JSONL digest and record count frozen in +`manifest.toml`. It validates raw records, leaf topology, effective logical +annotations, ColumnOrder union states, statistics field presence, and corpus +identity before it writes an atomic canonical JSONL result. + +`normalize_model.py` first regenerates the normalized raw bytes and requires an +exact match. It runs the hash-checked frozen model in an isolated Julia 1.10 or +1.12 process. Every column must produce a model result without a format error. +Only planned or verified case claims from `capabilities.toml` become PASS +records. Its run record binds the exact normalized raw input as upstream +evidence. `model-producer.toml` binds the model, suite, cases, bridge, and both +normalizer implementations as one producer revision. An unsupported claim is +never emitted as a pass. + +Both commands support `--check`. Check mode regenerates the complete payload in +memory and fails without changing the destination when the checked file is +absent or stale. Normal mode uses a same-directory temporary file, `fsync`, and +an atomic rename. Inputs and output aliases must be regular non-symlink files. + +Run the full self-test and freshness check without invoking the raw scanner: + +```sh +test/conformance/n6/normalizers/check.sh /absolute/path/to/raw.jsonl +``` + +Set `N6_MODEL_JULIA_EXECUTABLE` to the absolute `Sys.BINDIR/julia` path from +either authorized Julia 1.10.11 or Julia 1.12.6 toolchain. + +Check the generated-fixture evidence without running the scanner: + +```sh +test/conformance/n6/normalizers/check_generated.sh \ + /absolute/path/to/raw-generated.jsonl +``` diff --git a/test/conformance/n6/normalizers/check.sh b/test/conformance/n6/normalizers/check.sh new file mode 100755 index 0000000..2d7096d --- /dev/null +++ b/test/conformance/n6/normalizers/check.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +if [ "$#" -lt 1 ] || [ "$#" -gt 3 ]; then + echo "usage: check.sh RAW_JSONL [RAW_NORMALIZED] [MODEL_NORMALIZED]" >&2 + exit 2 +fi + +normalizers=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +n6=$(CDPATH= cd -- "$normalizers/.." && pwd) +raw_input=$1 +raw_output=${2:-$n6/evidence/raw-java-apache-corpus.normalized.jsonl} +model_output=${3:-$n6/evidence/independent-model.normalized.jsonl} +: "${N6_MODEL_JULIA_EXECUTABLE:?set N6_MODEL_JULIA_EXECUTABLE to a pinned absolute Julia executable}" + +python3 -B -I "$normalizers/runtests.py" +python3 -B -I "$normalizers/normalize_raw.py" \ + --input "$raw_input" --output "$raw_output" --check +python3 -B -I "$normalizers/normalize_model.py" \ + --input "$raw_output" --raw-input "$raw_input" \ + --output "$model_output" --julia-executable "$N6_MODEL_JULIA_EXECUTABLE" --check +python3 -B -I "$n6/validate_evidence.py" \ + --schema "$n6/evidence.schema.json" \ + --manifest "$n6/manifest.toml" \ + --capabilities "$n6/capabilities.toml" \ + --fixtures "$n6/fixtures.toml" \ + "$raw_output" "$model_output" diff --git a/test/conformance/n6/normalizers/check_generated.sh b/test/conformance/n6/normalizers/check_generated.sh new file mode 100755 index 0000000..9c1c5e8 --- /dev/null +++ b/test/conformance/n6/normalizers/check_generated.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + echo "usage: check_generated.sh RAW_JSONL [NORMALIZED_JSONL]" >&2 + exit 2 +fi + +normalizers=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +n6=$(CDPATH= cd -- "$normalizers/.." && pwd) +raw_input=$1 +output=${2:-$n6/evidence/raw-java-generated.normalized.jsonl} + +python3 -B -I "$normalizers/runtests.py" +python3 -B -I "$normalizers/normalize_raw.py" \ + --fixture-set generated --input "$raw_input" --output "$output" --check +python3 -B -I "$n6/validate_evidence.py" \ + --schema "$n6/evidence.schema.json" \ + --manifest "$n6/manifest.toml" \ + --capabilities "$n6/capabilities.toml" \ + --fixtures "$n6/fixtures.toml" \ + "$output" diff --git a/test/conformance/n6/normalizers/common.py b/test/conformance/n6/normalizers/common.py new file mode 100644 index 0000000..f59fa67 --- /dev/null +++ b/test/conformance/n6/normalizers/common.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.metadata +import json +import os +import pathlib +import stat +import sys +import tempfile +import tomllib + + +class EvidenceError(ValueError): + pass + + +def _open_regular(path, maximum): + path = pathlib.Path(path) + if not isinstance(maximum, int) or isinstance(maximum, bool) or maximum < 1: + raise ValueError("file byte limit must be a positive integer") + nofollow = getattr(os, "O_NOFOLLOW", None) + if nofollow is None: + raise EvidenceError("this platform cannot reject symbolic-link inputs") + flags = os.O_RDONLY | nofollow + flags |= getattr(os, "O_CLOEXEC", 0) + try: + descriptor = os.open(path, flags) + except FileNotFoundError as error: + raise EvidenceError(f"input is absent: {path}") from error + except OSError as error: + raise EvidenceError(f"input cannot be opened safely: {path}: {error}") from error + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + os.close(descriptor) + raise EvidenceError(f"input is not a regular file: {path}") + if not 0 < metadata.st_size <= maximum: + os.close(descriptor) + raise EvidenceError(f"input has an invalid byte size: {path}") + return descriptor, metadata + + +def _identity(metadata): + return (metadata.st_dev, metadata.st_ino, metadata.st_size, + metadata.st_mtime_ns, metadata.st_ctime_ns) + + +def read_file_bytes(path, maximum): + path = pathlib.Path(path) + descriptor, before = _open_regular(path, maximum) + try: + with os.fdopen(descriptor, "rb") as stream: + payload = stream.read(maximum + 1) + after = os.fstat(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + if len(payload) > maximum or len(payload) != before.st_size: + raise EvidenceError(f"input changed size while read: {path}") + if _identity(before) != _identity(after): + raise EvidenceError(f"input changed while read: {path}") + return payload + + +def _unique_object(path): + def hook(pairs): + value = {} + for key, item in pairs: + if key in value: + raise EvidenceError(f"{path}: duplicate JSON key: {key}") + value[key] = item + return value + return hook + + +def _invalid_constant(path): + def reject(value): + raise EvidenceError(f"{path}: invalid JSON constant: {value}") + return reject + + +def _invalid_float(path): + def reject(value): + raise EvidenceError(f"{path}: floating JSON number is forbidden: {value}") + return reject + + +def _canonical_integer(path): + def parse(value): + if value != "0" and (value.startswith("0") or value.startswith("-0")): + raise EvidenceError(f"{path}: noncanonical JSON integer: {value}") + return int(value) + return parse + + +def parse_jsonl_bytes(payload, label, max_line_bytes, max_records): + if not isinstance(payload, bytes): + raise TypeError("JSONL payload must be bytes") + if not payload: + raise EvidenceError(f"{label}: JSONL payload is empty") + if not isinstance(max_line_bytes, int) or isinstance(max_line_bytes, bool) or \ + max_line_bytes < 1: + raise ValueError("line byte limit must be a positive integer") + if not isinstance(max_records, int) or isinstance(max_records, bool) or \ + max_records < 1: + raise ValueError("record limit must be a positive integer") + path = pathlib.Path(label) + records = [] + start = 0 + while start < len(payload): + stop = payload.find(b"\n", start, start + max_line_bytes + 1) + number = len(records) + 1 + if stop < 0: + remaining = len(payload) - start + if remaining >= max_line_bytes: + raise EvidenceError(f"{path}:{number}: line exceeds its byte limit") + raise EvidenceError(f"{path}:{number}: missing final newline") + line = payload[start:stop + 1] + if len(line) > max_line_bytes: + raise EvidenceError(f"{path}:{number}: line exceeds its byte limit") + try: + decoded = line.decode("utf-8") + record = json.loads(decoded, + object_pairs_hook=_unique_object(path), + parse_constant=_invalid_constant(path), + parse_float=_invalid_float(path), + parse_int=_canonical_integer(path)) + except (UnicodeError, json.JSONDecodeError) as error: + raise EvidenceError(f"{path}:{number}: invalid JSON: {error}") from error + records.append(record) + if len(records) > max_records: + raise EvidenceError(f"{path}: record count exceeds its limit") + start = stop + 1 + return records + + +def read_jsonl(path, max_file_bytes, max_line_bytes, max_records): + payload = read_file_bytes(path, max_file_bytes) + return parse_jsonl_bytes(payload, path, max_line_bytes, max_records) + + +def _forbid_float(value): + if isinstance(value, float): + raise EvidenceError("floating JSON numbers are forbidden") + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise EvidenceError("JSON object keys must be strings") + _forbid_float(item) + elif isinstance(value, (list, tuple)): + for item in value: + _forbid_float(item) + return None + + +def canonical_json(value): + _forbid_float(value) + return json.dumps(value, ensure_ascii=False, allow_nan=False, + separators=(",", ":"), sort_keys=True) + + +def jsonl_bytes(records): + return b"".join((canonical_json(record) + "\n").encode("utf-8") + for record in records) + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path, maximum=32 * 1024 * 1024): + path = pathlib.Path(path) + descriptor, before = _open_regular(path, maximum) + digest = hashlib.sha256() + try: + with os.fdopen(descriptor, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + after = os.fstat(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + if _identity(before) != _identity(after): + raise EvidenceError(f"input changed while hashed: {path}") + return digest.hexdigest() + + +def ensure_snapshots(snapshots): + for path, expected in snapshots.items(): + if sha256_file(path) != expected: + raise EvidenceError(f"input changed after its snapshot: {path}") + return None + + +def capability_digest(case_id, capability_id, observations): + envelope = { + "capability_id": capability_id, + "case_id": case_id, + "observations": observations, + } + return sha256_bytes(canonical_json(envelope).encode("utf-8")) + + +def verify_python_toolchain(manifest): + matches = [item for item in manifest["toolchain"] + if item["id"] == "jsonschema-validator"] + if len(matches) != 1 or matches[0]["status"] != "verified": + raise EvidenceError("JSON Schema toolchain is not verified") + toolchain = matches[0] + artifacts = {item["name"]: item["sha256"] + for item in toolchain["artifacts"]} + if len(artifacts) != len(toolchain["artifacts"]): + raise EvidenceError("JSON Schema toolchain artifacts are ambiguous") + expected = artifacts.get("cpython-executable") + executable = pathlib.Path(sys.executable).resolve(strict=True) + if expected is None or sha256_file(executable) != expected: + raise EvidenceError("Python executable hash is not pinned") + versions = { + "jsonschema": "4.26.0", + "attrs": "25.4.0", + "jsonschema-specifications": "2025.9.1", + "referencing": "0.37.0", + "rpds-py": "0.30.0", + } + try: + actual = {name: importlib.metadata.version(name) for name in versions} + except importlib.metadata.PackageNotFoundError as error: + raise EvidenceError(f"pinned Python package is absent: {error}") from error + if actual != versions: + raise EvidenceError("Python package versions do not match the pin") + return None + + +def _reject_output_alias(output, inputs): + output = pathlib.Path(output) + output_resolved = output.resolve(strict=False) + for input_path in inputs: + input_path = pathlib.Path(input_path) + if output_resolved == input_path.resolve(strict=True): + raise EvidenceError(f"output aliases an input: {output}") + if output.exists() and os.path.samefile(output, input_path): + raise EvidenceError(f"output aliases an input: {output}") + return None + + +def _regular_output(path): + try: + metadata = pathlib.Path(path).stat(follow_symlinks=False) + except FileNotFoundError: + return False + if not stat.S_ISREG(metadata.st_mode): + raise EvidenceError(f"output is not a regular file: {path}") + return True + + +def publish_bytes(path, payload, check, inputs): + path = pathlib.Path(path) + if not isinstance(payload, bytes): + raise TypeError("output payload must be bytes") + parent = path.parent + if not parent.is_dir() or parent.is_symlink(): + raise EvidenceError(f"output parent is not a real directory: {parent}") + _reject_output_alias(path, inputs) + exists = _regular_output(path) + if check: + if not exists: + raise EvidenceError(f"freshness output is absent: {path}") + if read_file_bytes(path, max(1, len(payload))) != payload: + raise EvidenceError(f"freshness output differs: {path}") + return None + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=parent) + temporary = pathlib.Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as stream: + os.fchmod(stream.fileno(), 0o644) + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + _regular_output(path) + os.replace(temporary, path) + directory = os.open(parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + except BaseException: + temporary.unlink(missing_ok=True) + raise + return None + + +def parse_toml_bytes(payload, label): + try: + return tomllib.loads(payload.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise EvidenceError(f"invalid TOML input: {label}: {error}") from error + + +def load_toml(path, maximum=2 * 1024 * 1024): + return parse_toml_bytes(read_file_bytes(path, maximum), path) + + +def normalizer_root(): + return pathlib.Path(__file__).resolve(strict=True).parent + + +def n6_root(): + return normalizer_root().parent + + +def repository_root(): + root = n6_root().parents[2] + expected = root / "test" / "conformance" / "n6" + if expected != n6_root(): + raise EvidenceError("normalizer is outside the canonical repository layout") + return root diff --git a/test/conformance/n6/normalizers/model-producer.toml b/test/conformance/n6/normalizers/model-producer.toml new file mode 100644 index 0000000..d0bd29b --- /dev/null +++ b/test/conformance/n6/normalizers/model-producer.toml @@ -0,0 +1,34 @@ +descriptor_version = 1 +producer = "n6-independent-model" + +[[file]] +path = "test/conformance/n6/model/N6StatisticsModel.jl" +sha256 = "32c090ed6e0c6af49eabf3f96afc6e17dff630c4e89372367e693202e87c4262" + +[[file]] +path = "test/conformance/n6/model/README.md" +sha256 = "143290750312ae294c9954daf3d30d9d79b8627c67059ee001ba2228fa803814" + +[[file]] +path = "test/conformance/n6/model/cases.toml" +sha256 = "925da4abeb448033b291e8f2a1ed8d68f6af30e5a4994d1e576f757581c5ad7b" + +[[file]] +path = "test/conformance/n6/model/runtests.jl" +sha256 = "c74aab7a4d522a77443bba61070bae51c15bfec75a917bcd5ea16331d2307691" + +[[file]] +path = "test/conformance/n6/normalizers/common.py" +sha256 = "ea73b2731a981b8890baae47c756db2b42f0130926e100b91edace12fb2f0b0b" + +[[file]] +path = "test/conformance/n6/normalizers/model_bridge.jl" +sha256 = "2468012b4536136c9737629842bb37424d65c82dbc0c0303b74d2612f69afb23" + +[[file]] +path = "test/conformance/n6/normalizers/normalize_model.py" +sha256 = "51932b834d4c38147dc3452c615f09d45cbbc03b6bf69e2327dcdf9508a63296" + +[[file]] +path = "test/conformance/n6/normalizers/normalize_raw.py" +sha256 = "fb9a8c0bcca7a4ca0a252e735a891120af441b8861d3321b57ba6adf9ac3099a" diff --git a/test/conformance/n6/normalizers/model_bridge.jl b/test/conformance/n6/normalizers/model_bridge.jl new file mode 100644 index 0000000..cd24a77 --- /dev/null +++ b/test/conformance/n6/normalizers/model_bridge.jl @@ -0,0 +1,231 @@ +using SHA + +const INPUT_LIMIT = 16 * 1024 * 1024 +const MODEL_SHA256 = + "32c090ed6e0c6af49eabf3f96afc6e17dff630c4e89372367e693202e87c4262" +const MODEL_PATH = normpath(joinpath(@__DIR__, "..", "model", + "N6StatisticsModel.jl")) +const MODEL_TEST_PATH = normpath(joinpath(@__DIR__, "..", "model", + "runtests.jl")) + +bytes2hex(open(sha256, MODEL_PATH)) == MODEL_SHA256 || + error("independent model hash mismatch") +include(MODEL_PATH) +const Model = N6StatisticsModel + +function parsenothing(token::String, parser) + token == "-" && return nothing + return parser(token) +end + +function parsehex(token::String) + token == "-" && return nothing + iseven(ncodeunits(token)) || throw(ArgumentError("hex token has odd length")) + occursin(r"^[0-9a-f]*$", token) || + throw(ArgumentError("hex token is not lowercase hexadecimal")) + return hex2bytes(token) +end + +function parsebool(token::String) + token == "0" && return false + token == "1" && return true + throw(ArgumentError("Boolean token is invalid")) +end + +function physicaltype(token::String) + values = Dict( + "BOOLEAN" => Model.PHYSICAL_BOOLEAN, + "INT32" => Model.PHYSICAL_INT32, + "INT64" => Model.PHYSICAL_INT64, + "INT96" => Model.PHYSICAL_INT96, + "FLOAT" => Model.PHYSICAL_FLOAT, + "DOUBLE" => Model.PHYSICAL_DOUBLE, + "BYTE_ARRAY" => Model.PHYSICAL_BYTE_ARRAY, + "FIXED_LEN_BYTE_ARRAY" => Model.PHYSICAL_FIXED_LEN_BYTE_ARRAY, + ) + haskey(values, token) || throw(ArgumentError("physical type is invalid")) + return values[token] +end + +function logicaltype(token::String, signed::Union{Nothing,Bool}) + if token == "INTEGER" + signed === nothing && throw(ArgumentError("INTEGER lacks signedness")) + return signed ? Model.LOGICAL_SIGNED_INTEGER : + Model.LOGICAL_UNSIGNED_INTEGER + end + values = Dict( + "NONE" => Model.LOGICAL_NONE, + "STRING" => Model.LOGICAL_STRING, + "ENUM" => Model.LOGICAL_ENUM, + "JSON" => Model.LOGICAL_JSON, + "BSON" => Model.LOGICAL_BSON, + "UUID" => Model.LOGICAL_UUID, + "DECIMAL" => Model.LOGICAL_DECIMAL, + "DATE" => Model.LOGICAL_DATE, + "TIME" => Model.LOGICAL_TIME, + "TIMESTAMP" => Model.LOGICAL_TIMESTAMP, + "FLOAT16" => Model.LOGICAL_FLOAT16, + "INTERVAL" => Model.LOGICAL_INTERVAL, + "UNKNOWN" => Model.LOGICAL_UNKNOWN, + "VARIANT" => Model.LOGICAL_VARIANT, + "GEOMETRY" => Model.LOGICAL_GEOMETRY, + "GEOGRAPHY" => Model.LOGICAL_GEOGRAPHY, + "LIST" => Model.LOGICAL_LIST, + "MAP" => Model.LOGICAL_MAP, + ) + haskey(values, token) || throw(ArgumentError("logical type is invalid")) + return values[token] +end + +function timeunit(token::String) + token == "-" && return nothing + token == "MILLIS" && return Model.TIME_MILLIS + token == "MICROS" && return Model.TIME_MICROS + token == "NANOS" && return Model.TIME_NANOS + throw(ArgumentError("time unit is invalid")) +end + +function declaredorders(token::String, leaf::Int, count::Int) + token == "ABSENT" && return nothing + 0 <= leaf < count || throw(ArgumentError("leaf ordinal is invalid")) + orders = fill(Model.ORDER_FUTURE, count) + orders[leaf + 1] = token == "TYPE_ORDER" ? Model.ORDER_TYPE : + token == "IEEE_754_TOTAL_ORDER" ? Model.ORDER_IEEE : Model.ORDER_FUTURE + return orders +end + +function modelvalue(value) + value === nothing && return "NONE" + value isa Model.SignedValue && return "SIGNED:" * string(value.value) + value isa Model.UnsignedValue && return "UNSIGNED:" * string(value.value) + value isa Model.BooleanValue && return "BOOLEAN:" * (value.value ? "1" : "0") + value isa Model.ByteValue && return "BYTES:" * bytes2hex(value.value) + value isa Model.DecimalValue && return "DECIMAL:" * bytes2hex(value.value) + if value isa Model.FloatValue + width = Int(value.width) + digits = width ÷ 4 + return "FLOAT:" * string(width) * ":" * + string(value.bits; base=16, pad=digits) + end + throw(ArgumentError("model returned an unknown value kind")) +end + +function boundtokens(bound::Model.BoundFact) + return String[ + string(bound.state), + string(bound.reason), + string(bound.exactness), + modelvalue(bound.value), + ] +end + +function resulttokens(index::String, result::Model.StatisticsResult) + output = String[index, "OK"] + append!(output, boundtokens(result.lower)) + append!(output, boundtokens(result.upper)) + append!(output, String[ + result.null_count.known ? "1" : "0", + string(result.null_count.value), + result.nan_count.known ? "1" : "0", + string(result.nan_count.value), + result.distinct_count.known ? "1" : "0", + string(result.distinct_count.value), + string(result.occupancy), + string(result.family), + string(result.comparator), + string(result.trust.state), + string(result.trust.reason), + ]) + return output +end + +function createdby(present::Bool, token::String) + !present && token == "-" && return nothing + present || throw(ArgumentError("absent created_by carries bytes")) + bytes = parsehex(token) + bytes === nothing && throw(ArgumentError("created_by bytes are absent")) + return String(bytes) +end + +function interpret(fields::Vector{SubString{String}}) + length(fields) == 25 || throw(ArgumentError("bridge input field count is invalid")) + index = String(fields[1]) + leafindex = parse(Int, fields[4]) + leafcount = parse(Int, fields[5]) + signed = parsenothing(String(fields[10]), parsebool) + leaf = Model.LeafSpec( + physicaltype(String(fields[6])); + logical=logicaltype(String(fields[7]), signed), + type_length=parsenothing(String(fields[8]), token -> parse(Int, token)), + bit_width=parsenothing(String(fields[9]), token -> parse(Int, token)), + precision=parsenothing(String(fields[11]), token -> parse(Int, token)), + time_unit=timeunit(String(fields[12])), + ) + stats = Model.RawStatistics( + modern_lower=parsehex(String(fields[17])), + modern_upper=parsehex(String(fields[18])), + deprecated_lower=parsehex(String(fields[19])), + deprecated_upper=parsehex(String(fields[20])), + null_count=parsenothing(String(fields[21]), token -> parse(Int64, token)), + nan_count=parsenothing(String(fields[22]), token -> parse(Int64, token)), + distinct_count=parsenothing(String(fields[23]), token -> parse(Int64, token)), + lower_exact=parsenothing(String(fields[24]), parsebool), + upper_exact=parsenothing(String(fields[25]), parsebool), + ) + result = Model.interpret_statistics(leaf, parse(Int64, fields[13]), stats, + declaredorders(String(fields[14]), leafindex, leafcount); + leaf_index=leafindex + 1, leaf_count=leafcount, + created_by=createdby(parsebool(String(fields[15])), String(fields[16]))) + return resulttokens(index, result) +end + +function errorline(index::String, kind::String, error) + message = sprint(showerror, error) + return join(String[index, kind, bytes2hex(codeunits(message))], '\t') +end + +function runmodelsuite() + suite = Module(:N6FrozenModelSuite) + Core.eval(suite, :(include(path) = Base.include($suite, path))) + redirect_stdout(devnull) do + redirect_stderr(devnull) do + Base.include(suite, MODEL_TEST_PATH) + return + end + return + end + return +end + +function runbridge() + isempty(ARGS) || ARGS == ["--run-suite"] || + error("model bridge arguments are invalid") + input = read(stdin, INPUT_LIMIT + 1) + length(input) <= INPUT_LIMIT || error("bridge input exceeds its byte limit") + isempty(input) && error("bridge input is empty") + last(input) == 0x0a || error("bridge input lacks its final newline") + ARGS == ["--run-suite"] && runmodelsuite() + executable = Base.julia_cmd().exec[1] + println("TOOLCHAIN\t", VERSION, "\t", bytes2hex(open(sha256, executable))) + text = String(input) + lines = split(chop(text; tail=1), '\n'; keepempty=false) + length(lines) <= 313 || error("bridge input exceeds its record limit") + for line in lines + fields = split(line, '\t'; keepempty=true) + index = isempty(fields) ? "" : String(fields[1]) + try + println(join(interpret(fields), '\t')) + catch error + if error isa Model.ModelFormatError + println(errorline(index, "FORMAT_ERROR", error)) + elseif error isa ArgumentError + println(errorline(index, "ARGUMENT_ERROR", error)) + else + rethrow() + end + end + end + return +end + +runbridge() diff --git a/test/conformance/n6/normalizers/normalize_model.py b/test/conformance/n6/normalizers/normalize_model.py new file mode 100644 index 0000000..af0c1c1 --- /dev/null +++ b/test/conformance/n6/normalizers/normalize_model.py @@ -0,0 +1,815 @@ +#!/usr/bin/env python3 +import argparse +import contextlib +import hashlib +import jsonschema +import os +import pathlib +import re +import subprocess +import sys +import tempfile +import stat + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +import common +import normalize_raw + + +EVIDENCE_ID = "normalized-independent-model" +PRODUCER = "n6-independent-model" +SEMANTIC_CAPABILITIES = { + "semantic.type-order", + "semantic.ieee-total-order", + "semantic.logical-order", + "semantic.count-state", + "semantic.producer-trust", +} +BRIDGE_PATH = pathlib.Path(__file__).resolve().parent / "model_bridge.jl" +PRODUCER_DESCRIPTOR_PATH = pathlib.Path(__file__).resolve().parent / \ + "model-producer.toml" +MODEL_DIRECTORY = BRIDGE_PATH.parent.parent / "model" +MODEL_TEST_PATH = MODEL_DIRECTORY / "runtests.jl" +PRODUCER_FILE_LIMIT = 4 * 1024 * 1024 +BRIDGE_INPUT_LIMIT = 16 * 1024 * 1024 +BRIDGE_OUTPUT_LIMIT = 4 * 1024 * 1024 +INT64_MAX = (1 << 63) - 1 +BOUND_STATES = {"BOUND_ABSENT", "BOUND_UNKNOWN", "BOUND_KNOWN"} +EXACTNESS_STATES = { + "EXACTNESS_UNKNOWN", "EXACTNESS_INEXACT", "EXACTNESS_EXACT"} +BOUND_REASONS = { + "absent", "all_nan_type_order", "contradictory_bounds", + "deprecated_order_mismatch", "ieee_bound_kind_contradiction", + "invalid_logical", "known", "legacy_wrong_order", + "missing_column_orders", "nan_type_order", "no_non_null_values", + "over_limit", "parquet_251", "undefined_type_order", + "unknown_column_order", "unproven_ieee_nan", "widened_zero", +} +OCCUPANCY_STATES = { + "OCCUPANCY_UNKNOWN", "OCCUPANCY_EMPTY", "OCCUPANCY_ALL_NAN", + "OCCUPANCY_HAS_NON_NAN", +} +FAMILIES = {"FAMILY_NONE", "FAMILY_MODERN", "FAMILY_DEPRECATED"} +COMPARATORS = { + "COMPARATOR_SIGNED", "COMPARATOR_UNSIGNED", "COMPARATOR_UNSIGNED_BYTES", + "COMPARATOR_DECIMAL", "COMPARATOR_BOOLEAN", "COMPARATOR_TYPE_FLOAT", + "COMPARATOR_IEEE_FLOAT", "COMPARATOR_UNDEFINED", +} +TRUST_STATES = {"TRUST_TRUSTED", "TRUST_UNTRUSTED"} +TRUST_REASONS = {"no_bounds", "parquet_251", "legacy_wrong_order", "trusted"} +ALLOWED_JULIA_TOOLCHAINS = { + "1.10.11": { + "executable": + "6f687953e48958fc6596962379691d1c8a1720d3a9ff39c1e3113888e43bd8ae", + "runtime_tree": + "c784c03af8ab52e48aa6f57ce8aa06a3c9160671054301ab6c3dfebf2e582525", + }, + "1.12.6": { + "executable": + "9ad38bea81ecace044a4bdef2a0246dee94cb8a44c9420809cc00f9872651c64", + "runtime_tree": + "273ec71de498a36c77a7e4bb3af4a3f75c338bd1cfe255cab30805b6a2cda76e", + }, +} +JULIA_RUNTIME_MAX_ENTRIES = 10_000 +JULIA_RUNTIME_MAX_FILE_BYTES = 1024 * 1024 * 1024 +JULIA_RUNTIME_MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 + + +def _token(value): + if value is None: + return "-" + if isinstance(value, bool): + return "1" if value else "0" + return str(value) + + +def _hex_string(value): + if value is None: + return "-" + return value.encode("utf-8").hex() + + +def _model_input(index, column, file_record): + leaf = column["leaf_schema"] + fields = ( + index, + column["case_id"], + column["row_group"], + column["leaf"], + file_record["leaf_count"], + leaf["physical_type"], + leaf["logical_type"], + leaf["type_length"], + leaf["bit_width"], + leaf["is_signed"], + leaf["precision"], + leaf["time_unit"], + column["num_values"], + column["column_order"]["state"], + file_record["created_by_present"], + _hex_string(file_record["created_by"]), + column["min_value_hex"], + column["max_value_hex"], + column["deprecated_min_hex"], + column["deprecated_max_hex"], + column["null_count"], + column["nan_count"], + column["distinct_count"], + column["is_min_value_exact"], + column["is_max_value_exact"], + ) + tokens = [_token(value) for value in fields] + if any("\t" in value or "\n" in value for value in tokens): + raise common.EvidenceError("model bridge token contains a delimiter") + return "\t".join(tokens) + + +def _parse_value(token): + if token == "NONE": + return None + fields = token.split(":") + kind = fields[0] + if kind in ("SIGNED", "UNSIGNED") and len(fields) == 2: + value = int(fields[1]) + if kind == "SIGNED" and not -(1 << 63) <= value <= INT64_MAX: + raise common.EvidenceError("model returned an invalid signed value") + if kind == "UNSIGNED" and not 0 <= value < 1 << 64: + raise common.EvidenceError("model returned an invalid unsigned value") + return {"kind": kind, "value": fields[1]} + if kind == "BOOLEAN" and fields[1:] in (["0"], ["1"]): + return {"kind": kind, "value": fields[1] == "1"} + if kind in ("BYTES", "DECIMAL") and len(fields) == 2: + if re.fullmatch(r"(?:[0-9a-f]{2})*", fields[1]) is None: + raise common.EvidenceError("model returned invalid hexadecimal bytes") + return {"kind": kind, "hex": fields[1]} + if kind == "FLOAT" and len(fields) == 3: + width = int(fields[1]) + if width not in (16, 32, 64) or len(fields[2]) != width // 4: + raise common.EvidenceError("model bridge returned an invalid float") + if re.fullmatch(r"[0-9a-f]+", fields[2]) is None: + raise common.EvidenceError("model returned invalid float bits") + return {"kind": kind, "width": width, "bits_hex": fields[2]} + raise common.EvidenceError("model bridge returned an invalid value") + + +def _parse_bound(fields, offset): + result = { + "state": fields[offset], + "reason": fields[offset + 1], + "exactness": fields[offset + 2], + "value": _parse_value(fields[offset + 3]), + } + if result["state"] not in BOUND_STATES or \ + result["reason"] not in BOUND_REASONS or \ + result["exactness"] not in EXACTNESS_STATES: + raise common.EvidenceError("model bridge returned an invalid bound fact") + known = result["state"] == "BOUND_KNOWN" + if known == (result["value"] is None): + raise common.EvidenceError("model bound state contradicts its value") + return result + + +def _parse_ok(fields): + if len(fields) != 21: + raise common.EvidenceError("model bridge output field count is invalid") + booleans = (fields[10], fields[12], fields[14]) + if any(value not in ("0", "1") for value in booleans): + raise common.EvidenceError("model bridge returned an invalid count state") + counts = [int(value) for value in (fields[11], fields[13], fields[15])] + if any(not 0 <= value <= INT64_MAX for value in counts): + raise common.EvidenceError("model bridge returned an invalid count") + if any(fields[index] == "0" and fields[index + 1] != "0" + for index in (10, 12, 14)): + raise common.EvidenceError("unknown model count has a value") + if fields[16] not in OCCUPANCY_STATES or fields[17] not in FAMILIES or \ + fields[18] not in COMPARATORS or fields[19] not in TRUST_STATES or \ + fields[20] not in TRUST_REASONS: + raise common.EvidenceError("model bridge returned an invalid state") + return { + "outcome": "OK", + "lower": _parse_bound(fields, 2), + "upper": _parse_bound(fields, 6), + "counts": { + "null": {"known": fields[10] == "1", "value": fields[11]}, + "nan": {"known": fields[12] == "1", "value": fields[13]}, + "distinct": {"known": fields[14] == "1", "value": fields[15]}, + }, + "occupancy": fields[16], + "family": fields[17], + "comparator": fields[18], + "trust": {"state": fields[19], "reason": fields[20]}, + } + + +def _parse_result(line): + fields = line.split("\t") + if len(fields) < 2: + raise common.EvidenceError("model bridge returned a short line") + index = fields[0] + if fields[1] == "OK": + return index, _parse_ok(fields) + if fields[1] in ("FORMAT_ERROR", "ARGUMENT_ERROR") and len(fields) == 3: + try: + message = bytes.fromhex(fields[2]).decode("utf-8") + except (ValueError, UnicodeError) as error: + raise common.EvidenceError( + "model bridge returned an invalid error message") from error + return index, {"outcome": fields[1], "message": message} + raise common.EvidenceError("model bridge returned an invalid outcome") + + +def _julia_runtime_tree_sha256(root): + root = pathlib.Path(root) + if root.resolve(strict=True) != root or root.is_symlink() or not root.is_dir(): + raise common.EvidenceError("Julia runtime root is not canonical") + entries = [] + total_bytes = 0 + for directory, directories, files in os.walk(root, followlinks=False): + base = pathlib.Path(directory) + for name in directories + files: + path = base / name + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode) or stat.S_ISREG(metadata.st_mode): + relative = path.relative_to(root).as_posix() + if "\0" in relative or "\n" in relative: + raise common.EvidenceError("Julia runtime path is invalid") + entries.append((relative, path, metadata)) + if stat.S_ISREG(metadata.st_mode): + if not 0 <= metadata.st_size <= JULIA_RUNTIME_MAX_FILE_BYTES: + raise common.EvidenceError( + "Julia runtime file has an invalid size") + total_bytes += metadata.st_size + elif not stat.S_ISDIR(metadata.st_mode): + raise common.EvidenceError("Julia runtime has a special file") + if len(entries) > JULIA_RUNTIME_MAX_ENTRIES or \ + total_bytes > JULIA_RUNTIME_MAX_TOTAL_BYTES: + raise common.EvidenceError("Julia runtime tree exceeds its limit") + digest = hashlib.sha256() + for relative, path, metadata in sorted(entries): + encoded = relative.encode("utf-8") + if stat.S_ISLNK(metadata.st_mode): + target = os.readlink(path) + if "\0" in target or "\n" in target: + raise common.EvidenceError("Julia runtime link is invalid") + record = b"L\0" + encoded + b"\0" + target.encode("utf-8") + b"\n" + else: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + nofollow = getattr(os, "O_NOFOLLOW", None) + if nofollow is None: + raise common.EvidenceError( + "platform cannot reject Julia runtime symlinks") + descriptor = os.open(path, flags | nofollow) + file_digest = hashlib.sha256() + try: + with os.fdopen(descriptor, "rb") as stream: + before = os.fstat(stream.fileno()) + if not stat.S_ISREG(before.st_mode) or \ + not 0 <= before.st_size <= \ + JULIA_RUNTIME_MAX_FILE_BYTES: + raise common.EvidenceError( + "Julia runtime file has an invalid size") + for block in iter(lambda: stream.read(1024 * 1024), b""): + file_digest.update(block) + after = os.fstat(stream.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + identity = lambda value: (value.st_dev, value.st_ino, + value.st_size, value.st_mtime_ns, value.st_ctime_ns) + if identity(before) != identity(after): + raise common.EvidenceError( + "Julia runtime file changed while hashed") + record = b"F\0" + encoded + b"\0" + \ + file_digest.hexdigest().encode("ascii") + b"\n" + digest.update(record) + return digest.hexdigest() + + +def _julia_toolchain(command): + if not isinstance(command, (list, tuple)) or len(command) != 1 or \ + not isinstance(command[0], str): + raise common.EvidenceError( + "Julia model command must be one absolute executable") + executable = pathlib.Path(command[0]) + if not executable.is_absolute() or executable.resolve(strict=True) != executable: + raise common.EvidenceError( + "Julia model executable must be absolute and canonical") + digest = common.sha256_file(executable, 2 * 1024 * 1024) + matches = [version for version, expected in ALLOWED_JULIA_TOOLCHAINS.items() + if digest == expected["executable"]] + if len(matches) != 1: + raise common.EvidenceError("Julia model executable is not pinned") + version = matches[0] + runtime_root = executable.parent.parent + runtime_digest = _julia_runtime_tree_sha256(runtime_root) + if runtime_digest != ALLOWED_JULIA_TOOLCHAINS[version]["runtime_tree"]: + raise common.EvidenceError("Julia model runtime tree is not pinned") + return executable, runtime_root, version, digest, runtime_digest + + +def run_model(columns, files, command, producer_root, run_suite=False): + executable, runtime_root, expected_version, expected_digest, \ + expected_runtime_digest = _julia_toolchain(command) + ordered = sorted(columns, + key=lambda item: (item["case_id"], item["row_group"], item["leaf"])) + keys = [] + lines = [] + for index, column in enumerate(ordered): + case_id = column["case_id"] + if case_id not in files: + raise common.EvidenceError("model column lacks its file record") + token = str(index) + keys.append((case_id, column["row_group"], column["leaf"])) + lines.append(_model_input(token, column, files[case_id])) + payload = ("\n".join(lines) + "\n").encode("ascii") + if not 0 < len(payload) <= BRIDGE_INPUT_LIMIT: + raise common.EvidenceError("model bridge input exceeds its byte limit") + bridge_path = pathlib.Path(producer_root) / \ + "test/conformance/n6/normalizers/model_bridge.jl" + arguments = [str(executable), + "--startup-file=no", + "--history-file=no", + "--compiled-modules=no", + "--check-bounds=yes", + "--project=@stdlib", + str(bridge_path), + ] + if run_suite: + arguments.append("--run-suite") + environment = os.environ.copy() + environment["JULIA_LOAD_PATH"] = "@stdlib" + environment["JULIA_PROJECT"] = "@stdlib" + environment["JULIA_NUM_THREADS"] = "1" + with tempfile.TemporaryDirectory(prefix="parquet-n6-model-depot-") as depot: + environment["JULIA_DEPOT_PATH"] = depot + try: + process = subprocess.run(arguments, input=payload, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120, + check=False, env=environment) + except (OSError, subprocess.TimeoutExpired) as error: + raise common.EvidenceError(f"independent model failed: {error}") from error + if len(process.stdout) > BRIDGE_OUTPUT_LIMIT or \ + len(process.stderr) > BRIDGE_OUTPUT_LIMIT: + raise common.EvidenceError("independent model output exceeds its byte limit") + if process.returncode != 0: + detail = process.stderr.decode("utf-8", errors="replace") + raise common.EvidenceError( + f"independent model exited {process.returncode}: {detail}") + try: + text = process.stdout.decode("utf-8") + except UnicodeError as error: + raise common.EvidenceError("independent model output is not UTF-8") from error + if not text.endswith("\n"): + raise common.EvidenceError("independent model output lacks a final newline") + output_lines = text[:-1].split("\n") + if len(output_lines) != len(lines) + 1: + raise common.EvidenceError("independent model output count is inconsistent") + toolchain = output_lines[0].split("\t") + if len(toolchain) != 3 or toolchain[0] != "TOOLCHAIN" or \ + (toolchain[1], toolchain[2]) != \ + (expected_version, expected_digest): + raise common.EvidenceError("independent model toolchain line is invalid") + if common.sha256_file(executable, 2 * 1024 * 1024) != expected_digest: + raise common.EvidenceError("Julia model executable changed after use") + if _julia_runtime_tree_sha256(runtime_root) != expected_runtime_digest: + raise common.EvidenceError("Julia model runtime tree changed after use") + results = {} + for line in output_lines[1:]: + index, result = _parse_result(line) + try: + position = int(index) + except (ValueError, IndexError) as error: + raise common.EvidenceError("independent model index is invalid") from error + if not 0 <= position < len(keys): + raise common.EvidenceError("independent model index is invalid") + key = keys[position] + if key in results: + raise common.EvidenceError("independent model returned a duplicate result") + results[key] = result + if len(results) != len(keys): + raise common.EvidenceError("independent model result coverage is incomplete") + return expected_runtime_digest, results + + +def require_model_success(results): + failures = sorted((key, value) for key, value in results.items() + if value["outcome"] != "OK") + if failures: + key, result = failures[0] + raise common.EvidenceError( + f"independent model rejected normalized column {key}: " + f"{result['outcome']}: {result['message']}") + return None + + +def _verify_models(context): + root = context["root"] + expected_models = { + "test/conformance/n6/model/N6StatisticsModel.jl", + "test/conformance/n6/model/runtests.jl", + "test/conformance/n6/model/cases.toml", + "test/conformance/n6/model/README.md", + } + actual_models = { + item["file"] for item in context["manifest"]["frozen_model"]} + if actual_models != expected_models or len(actual_models) != len( + context["manifest"]["frozen_model"]): + raise common.EvidenceError("frozen independent model set is inconsistent") + model_hashes = {} + model_payloads = {} + for item in context["manifest"]["frozen_model"]: + path = root / item["file"] + payload = common.read_file_bytes(path, PRODUCER_FILE_LIMIT) + digest = common.sha256_bytes(payload) + if digest != item["sha256"]: + raise common.EvidenceError( + f"frozen independent model hash mismatch: {item['file']}") + model_hashes[item["file"]] = digest + model_payloads[path] = payload + descriptor_relative = context["manifest"].get( + "model_producer_descriptor_file") + descriptor_expected = context["manifest"].get( + "model_producer_descriptor_sha256") + expected_descriptor = \ + "test/conformance/n6/normalizers/model-producer.toml" + if descriptor_relative != expected_descriptor or \ + not isinstance(descriptor_expected, str): + raise common.EvidenceError("model producer descriptor is inconsistent") + descriptor_path = root / descriptor_relative + if descriptor_path != PRODUCER_DESCRIPTOR_PATH: + raise common.EvidenceError("model producer descriptor path differs") + descriptor_payload = common.read_file_bytes(descriptor_path, 2 * 1024 * 1024) + descriptor_sha256 = common.sha256_bytes(descriptor_payload) + if descriptor_sha256 != descriptor_expected: + raise common.EvidenceError("model producer descriptor hash mismatch") + descriptor = common.parse_toml_bytes(descriptor_payload, descriptor_path) + model_payloads[descriptor_path] = descriptor_payload + if set(descriptor) != {"descriptor_version", "producer", "file"} or \ + descriptor["descriptor_version"] != 1 or \ + descriptor["producer"] != PRODUCER or \ + not isinstance(descriptor["file"], list): + raise common.EvidenceError("model producer descriptor header differs") + expected_producer_files = { + *expected_models, + "test/conformance/n6/normalizers/common.py", + "test/conformance/n6/normalizers/model_bridge.jl", + "test/conformance/n6/normalizers/normalize_model.py", + "test/conformance/n6/normalizers/normalize_raw.py", + } + producer_hashes = {} + for item in descriptor["file"]: + if not isinstance(item, dict) or set(item) != {"path", "sha256"} or \ + not isinstance(item["path"], str) or \ + not isinstance(item["sha256"], str): + raise common.EvidenceError("model producer file entry differs") + relative = pathlib.PurePosixPath(item["path"]) + if relative.is_absolute() or ".." in relative.parts or \ + relative.as_posix() != item["path"]: + raise common.EvidenceError("model producer path is unsafe") + path = root.joinpath(*relative.parts) + payload = model_payloads.get(path) + if payload is None: + payload = common.read_file_bytes(path, PRODUCER_FILE_LIMIT) + model_payloads[path] = payload + digest = common.sha256_bytes(payload) + if digest != item["sha256"]: + raise common.EvidenceError( + f"model producer file hash mismatch: {item['path']}") + if item["path"] in producer_hashes: + raise common.EvidenceError("model producer file is duplicated") + producer_hashes[item["path"]] = digest + if set(producer_hashes) != expected_producer_files: + raise common.EvidenceError("model producer file set is inconsistent") + if any(producer_hashes[path] != digest + for path, digest in model_hashes.items()): + raise common.EvidenceError("model producer and frozen model differ") + authority = _authority(context) + if authority["revision"] != descriptor_sha256: + raise common.EvidenceError("model authority revision is inconsistent") + toolchains = {item["id"]: item for item in context["manifest"]["toolchain"]} + if len(toolchains) != len(context["manifest"]["toolchain"]): + raise common.EvidenceError("manifest toolchain IDs are ambiguous") + expected_hashes = set() + for version, digests in ALLOWED_JULIA_TOOLCHAINS.items(): + identifier = "julia-" + ".".join(version.split(".")[:2]) + toolchain = toolchains.get(identifier) + expected_artifacts = [{ + "name": f"julia-{version}-executable", + "sha256": digests["executable"], + }, { + "name": f"julia-{version}-runtime-tree-sha256-v1", + "sha256": digests["runtime_tree"], + }] + if toolchain is None or toolchain["status"] != "verified" or \ + toolchain["version"] != version or \ + toolchain["artifacts"] != expected_artifacts: + raise common.EvidenceError( + f"Julia model toolchain is inconsistent: {identifier}") + expected_hashes.update(digests.values()) + if set(authority["toolchain_sha256"]) != expected_hashes or \ + len(authority["toolchain_sha256"]) != len(expected_hashes): + raise common.EvidenceError("model authority toolchains are inconsistent") + context["model_snapshots"] = { + root / relative: digest for relative, digest in producer_hashes.items()} + context["model_snapshots"][descriptor_path] = descriptor_sha256 + context["model_payloads"] = model_payloads + return None + + +def _set_snapshot_modes(root, locked): + root = pathlib.Path(root) + directories = [] + for directory, names, files in os.walk(root, topdown=True, + followlinks=False): + path = pathlib.Path(directory) + directories.append(path) + for name in files: + target = path / name + if target.is_symlink(): + raise common.EvidenceError( + f"model snapshot file is a symbolic link: {target}") + target.chmod(0o400 if locked else 0o600) + for name in names: + target = path / name + if target.is_symlink(): + raise common.EvidenceError( + f"model snapshot directory is a symbolic link: {target}") + for directory in reversed(directories): + directory.chmod(0o500 if locked else 0o700) + return None + + +@contextlib.contextmanager +def model_producer_snapshot(context): + payloads = context.get("model_payloads") + if not isinstance(payloads, dict) or not payloads: + raise common.EvidenceError("model producer bytes are not authenticated") + with tempfile.TemporaryDirectory( + prefix="parquet-n6-model-producer-") as directory: + snapshot_root = pathlib.Path(directory) / "producer" + snapshot_root.mkdir(mode=0o700) + for source, payload in payloads.items(): + try: + relative = pathlib.Path(source).relative_to(context["root"]) + except ValueError as error: + raise common.EvidenceError( + f"model producer path escapes the repository: {source}") from error + target = snapshot_root / relative + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with target.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + _set_snapshot_modes(snapshot_root, True) + try: + yield snapshot_root + finally: + _set_snapshot_modes(snapshot_root, False) + + +def _semantic_cases(context): + path = context["root"] / "test/conformance/n6/model/cases.toml" + value = context["model_payloads"][path] + cases = common.parse_toml_bytes(value, path) + if cases.get("schema_version") != 1 or \ + not isinstance(cases.get("case_groups"), list): + raise common.EvidenceError("semantic case manifest header differs") + output = {} + for case in cases["case_groups"]: + required = {"id", "requirements", "capabilities", "digest_contract", + "expected_sha256"} + if not isinstance(case, dict) or set(case) != required: + raise common.EvidenceError("semantic case fields differ") + identifier = case["id"] + if identifier in output: + raise common.EvidenceError("semantic case ID is duplicated") + output[identifier] = case + return output + + +def _semantic_observations(case, context): + snapshots = context["model_snapshots"] + model_path = context["root"] / \ + "test/conformance/n6/model/N6StatisticsModel.jl" + tests_path = context["root"] / "test/conformance/n6/model/runtests.jl" + return [{ + "case_group": case["id"], + "contract": "n6-independent-model-suite-v1", + "model_sha256": snapshots[model_path], + "requirements": case["requirements"], + "runtests_sha256": snapshots[tests_path], + "suite_status": "PASS", + }] + + +def _raw_records(normalized_path, raw_path, context): + limits = context["manifest"]["evidence_limits"] + payload = common.read_file_bytes(normalized_path, limits["max_file_bytes"]) + expected = normalize_raw.render_raw_evidence(raw_path) + if payload != expected: + raise common.EvidenceError( + "normalized raw input is not the deterministic frozen-corpus result") + normalize_raw.validate_normalized_payload(payload, context) + records = common.parse_jsonl_bytes(payload, normalized_path, + limits["max_line_bytes"], limits["max_records_per_input"]) + for record in records: + context["normalized_validator"].validate(record) + return records, payload + + +def _raw_upstream(context, payload): + entries = [item for section in ("frozen_evidence", "planned_evidence") + for item in context["manifest"].get(section, []) + if item["id"] == normalize_raw.EVIDENCE_ID] + if len(entries) != 1: + raise common.EvidenceError("normalized raw evidence entry is ambiguous") + entry = entries[0] + if entry.get("authority") != normalize_raw.PRODUCER or \ + entry.get("format") != "normalized-jsonl": + raise common.EvidenceError("normalized raw evidence entry is inconsistent") + return [{ + "evidence_id": normalize_raw.EVIDENCE_ID, + "file": entry["file"], + "sha256": common.sha256_bytes(payload), + }] + + +def _authority(context): + matches = [item for item in context["capabilities"]["authority"] + if item["id"] == PRODUCER] + if len(matches) != 1: + raise common.EvidenceError("independent model authority is ambiguous") + authority = matches[0] + if authority["kind"] != "test-owned-independent-semantics" or \ + authority["version"] != "1": + raise common.EvidenceError("independent model authority is inconsistent") + return authority + + +def _model_claims(context): + claims = {} + for claim in _authority(context).get("claim", []): + if claim["capability"] not in SEMANTIC_CAPABILITIES or \ + claim["status"] not in ("verified", "planned"): + continue + for case_id in claim["cases"]: + key = (case_id, claim["capability"]) + if key in claims: + raise common.EvidenceError(f"duplicate model claim: {key}") + claims[key] = claim["status"] + return claims + + +def _run_record(context, toolchain, upstream_evidence): + authority = _authority(context) + if toolchain not in authority["toolchain_sha256"]: + raise common.EvidenceError( + "independent model executable is not an allowed toolchain") + manifest = context["manifest"] + paths = context["paths"] + return { + "record": "run", + "schema_version": 2, + "evidence_id": EVIDENCE_ID, + "producer": PRODUCER, + "producer_version": authority["version"], + "source_revision": authority["revision"], + "plan_sha256": manifest["plan_sha256"], + "capabilities_sha256": context["snapshots"][paths["capabilities"]], + "fixture_manifest_sha256": context["snapshots"][paths["fixtures"]], + "corpus_manifest_sha256": context["snapshots"][paths["corpus"]], + "evidence_schema_sha256": context["snapshots"][paths["schema"]], + "toolchain_sha256": toolchain, + "upstream_evidence": upstream_evidence, + "unsupported_cases": [], + } + + +def _result(case, capability, observations, detail=None): + digest = common.capability_digest(case["id"], capability, observations) + expected = case.get("expected_sha256", {}).get(capability, digest) + if digest != expected: + raise common.EvidenceError( + f"semantic case digest differs: {(case['id'], capability)}: " + f"expected={expected}, actual={digest}") + return { + "record": "case_result", + "schema_version": 2, + "case_id": case["id"], + "capability_id": capability, + "digest_contract": case.get( + "digest_contract", "n6-capability-result-sha256-v1"), + "status": "PASS", + "expected_sha256": expected, + "actual_sha256": digest, + "detail": detail or + "Frozen independent model interpreted every normalized raw column in this case.", + } + + +def render_model_evidence(normalized_path, raw_path, julia_command): + context = normalize_raw.load_context() + _verify_models(context) + records, raw_payload = _raw_records(normalized_path, raw_path, context) + files = {record["case_id"]: record for record in records + if record["record"] == "file"} + columns = [record for record in records + if record["record"] == "column_statistics"] + cases = [case for case in context["fixtures"]["fixture"] + if case["source_kind"] == "apache-corpus"] + if len(files) != context["raw_entry"]["case_count"] or \ + set(files) != {case["id"] for case in cases}: + raise common.EvidenceError("normalized raw file coverage is incomplete") + with model_producer_snapshot(context) as producer_root: + toolchain, model_results = run_model(columns, files, julia_command, + producer_root, run_suite=True) + require_model_success(model_results) + claims = _model_claims(context) + semantic_cases = _semantic_cases(context) + output = [_run_record(context, toolchain, + _raw_upstream(context, raw_payload))] + emitted_semantic = set() + for case_id in sorted(semantic_cases): + case = semantic_cases[case_id] + for capability in case["capabilities"]: + if (case_id, capability) not in claims: + continue + emitted_semantic.add((case_id, capability)) + output.append(_result(case, capability, + _semantic_observations(case, context), + "The frozen independent model suite passed this semantic case.")) + expected_semantic = {key for key in claims if key[0] in semantic_cases} + if emitted_semantic != expected_semantic: + raise common.EvidenceError("semantic model claim coverage is incomplete") + for case in cases: + case_id = case["id"] + file_record = files[case_id] + case_columns = sorted((record for record in columns + if record["case_id"] == case_id), + key=lambda record: (record["row_group"], record["leaf"])) + output.append(file_record) + output.extend(case_columns) + observations = [{ + "file": file_record, + "columns": [{ + "column": record, + "model_result": model_results[( + case_id, record["row_group"], record["leaf"])], + } for record in case_columns], + }] + capabilities = sorted(capability for capability in case["capabilities"] + if (case_id, capability) in claims) + output.extend(_result(case, capability, observations) + for capability in capabilities) + for record in output: + context["normalized_validator"].validate(record) + payload = common.jsonl_bytes(output) + limits = context["manifest"]["evidence_limits"] + if len(payload) > limits["max_file_bytes"] or \ + len(output) > limits["max_records_per_input"]: + raise common.EvidenceError("model evidence exceeds a frozen limit") + normalize_raw.validate_normalized_payload(payload, context, (raw_payload,)) + return payload + + +def parse_arguments(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True, + help="deterministic normalized raw evidence") + parser.add_argument("--raw-input", required=True, + help="frozen gate-generated raw scanner output") + parser.add_argument("--output", required=True) + parser.add_argument("--julia-executable", required=True) + parser.add_argument("--check", action="store_true") + return parser.parse_args(argv) + + +def main(argv=None): + try: + args = parse_arguments(argv) + command = [args.julia_executable] + payload = render_model_evidence( + args.input, args.raw_input, command) + context = normalize_raw.load_context() + _verify_models(context) + common.ensure_snapshots(context["snapshots"]) + common.ensure_snapshots(context["model_snapshots"]) + inputs = (args.input, args.raw_input, *context["paths"].values(), + *context["model_snapshots"]) + common.publish_bytes(args.output, payload, args.check, inputs) + action = "is fresh" if args.check else "written" + print(f"N6 independent model evidence {action}: {args.output}") + return 0 + except (common.EvidenceError, OSError, ValueError, + jsonschema.ValidationError) as error: + print(f"N6 model normalization failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/conformance/n6/normalizers/normalize_raw.py b/test/conformance/n6/normalizers/normalize_raw.py new file mode 100644 index 0000000..11c6732 --- /dev/null +++ b/test/conformance/n6/normalizers/normalize_raw.py @@ -0,0 +1,808 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import pathlib +import re +import subprocess +import sys +import tempfile + +import jsonschema + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +import common + + +INT64_MAX = (1 << 63) - 1 +INT32_MIN = -(1 << 31) +INT32_MAX = (1 << 31) - 1 +FORMAT_COMMIT = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" +RAW_EVIDENCE_ID = "raw-java-apache-corpus" +EVIDENCE_ID = "normalized-raw-java-apache-corpus" +GENERATED_RAW_EVIDENCE_ID = "raw-java-generated" +GENERATED_EVIDENCE_ID = "normalized-raw-java-generated" +GENERATED_RAW_SHA256 = \ + "4f5e896a53c970cb3ae03876d40dafb202850929e4dc6b589aece474b0d61a25" +GENERATED_RAW_CASE_COUNT = 12 +GENERATED_RAW_RECORD_COUNT = 12 +PRODUCER = "n6-raw-java" +WIRE_CAPABILITIES = { + "wire.column-order.type", + "wire.column-order.ieee", + "wire.column-order.empty", + "wire.statistics.deprecated-bounds", + "wire.statistics.modern-bounds", + "wire.statistics.exactness", + "wire.statistics.counts", + "wire.statistics.nan-count", +} +CONVERTED_LOGICAL = { + "UTF8": "STRING", + "ENUM": "ENUM", + "DECIMAL": "DECIMAL", + "DATE": "DATE", + "TIME_MILLIS": "TIME", + "TIME_MICROS": "TIME", + "TIMESTAMP_MILLIS": "TIMESTAMP", + "TIMESTAMP_MICROS": "TIMESTAMP", + "UINT_8": "INTEGER", + "UINT_16": "INTEGER", + "UINT_32": "INTEGER", + "UINT_64": "INTEGER", + "INT_8": "INTEGER", + "INT_16": "INTEGER", + "INT_32": "INTEGER", + "INT_64": "INTEGER", + "JSON": "JSON", + "BSON": "BSON", + "INTERVAL": "INTERVAL", +} +INTEGER_CONVERTED = { + "UINT_8": (8, False), + "UINT_16": (16, False), + "UINT_32": (32, False), + "UINT_64": (64, False), + "INT_8": (8, True), + "INT_16": (16, True), + "INT_32": (32, True), + "INT_64": (64, True), +} +TIME_CONVERTED = { + "TIME_MILLIS": ("TIME", "MILLIS"), + "TIME_MICROS": ("TIME", "MICROS"), + "TIMESTAMP_MILLIS": ("TIMESTAMP", "MILLIS"), + "TIMESTAMP_MICROS": ("TIMESTAMP", "MICROS"), +} +GROUP_TYPES = {"MAP", "MAP_KEY_VALUE", "LIST", "VARIANT"} +STATISTIC_FIELDS = ( + (1, "max", "deprecated_max_hex"), + (2, "min", "deprecated_min_hex"), + (3, "null_count", "null_count"), + (4, "distinct_count", "distinct_count"), + (5, "max_value", "max_value_hex"), + (6, "min_value", "min_value_hex"), + (7, "is_max_value_exact", "is_max_value_exact"), + (8, "is_min_value_exact", "is_min_value_exact"), + (9, "nan_count", "nan_count"), +) +CORPUS_LINE = re.compile( + rb"([0-9a-f]{64}) (data/[A-Za-z0-9._+@=-]+(?:/[A-Za-z0-9._+@=-]+)*)\n") + + +def _presence(raw, name): + value = raw[name] + return value["value"] if value["present"] else None + + +def _required_converted(logical, bit_width, is_signed, time_unit): + direct = { + "STRING": "UTF8", + "ENUM": "ENUM", + "DECIMAL": "DECIMAL", + "DATE": "DATE", + "JSON": "JSON", + "BSON": "BSON", + "INTERVAL": "INTERVAL", + } + if logical in direct: + return direct[logical] + if logical == "INTEGER" and bit_width is not None and is_signed is not None: + return ("INT_" if is_signed else "UINT_") + str(bit_width) + if logical in ("TIME", "TIMESTAMP") and time_unit in ("MILLIS", "MICROS"): + return logical + "_" + time_unit + return None + + +def effective_leaf(raw): + physical = raw["physical_type"] + type_length = _presence(raw, "type_length") + converted = _presence(raw, "converted_type") + raw_scale = _presence(raw, "scale") + raw_precision = _presence(raw, "precision") + logical_union = raw["logical_type"] + bit_width = None + is_signed = None + time_unit = None + adjusted = None + crs = None + algorithm = None + scale = None + precision = None + if logical_union["present"]: + logical = logical_union["member"] + if logical is None: + raise common.EvidenceError( + "future logical union member cannot be normalized") + if logical in GROUP_TYPES: + raise common.EvidenceError( + f"group logical type is present on a leaf: {logical}") + parameters = logical_union["parameters"] + if logical == "INTEGER": + bit_width = parameters["bit_width"] + is_signed = parameters["is_signed"] + elif logical == "DECIMAL": + scale = parameters["scale"] + precision = parameters["precision"] + if raw_scale != scale or raw_precision != precision: + raise common.EvidenceError( + "modern DECIMAL contradicts SchemaElement parameters") + elif logical in ("TIME", "TIMESTAMP"): + time_unit = parameters["unit"] + adjusted = parameters["is_adjusted_to_utc"] + elif logical == "GEOMETRY": + crs = parameters["crs"]["value"] if \ + parameters["crs"]["present"] else None + elif logical == "GEOGRAPHY": + crs = parameters["crs"]["value"] if \ + parameters["crs"]["present"] else None + algorithm = parameters["algorithm"]["value"] if \ + parameters["algorithm"]["present"] else None + else: + if converted in GROUP_TYPES: + raise common.EvidenceError( + f"group converted type is present on a leaf: {converted}") + logical = CONVERTED_LOGICAL.get(converted, "NONE") + if converted == "DECIMAL": + if raw_scale is None or raw_precision is None: + raise common.EvidenceError( + "legacy DECIMAL lacks precision or scale") + scale = raw_scale + precision = raw_precision + elif converted in INTEGER_CONVERTED: + bit_width, is_signed = INTEGER_CONVERTED[converted] + elif converted in TIME_CONVERTED: + expected, time_unit = TIME_CONVERTED[converted] + if logical != expected: + raise common.EvidenceError("legacy temporal type is inconsistent") + adjusted = True + if logical != "DECIMAL" and (raw_scale is not None or raw_precision is not None): + raise common.EvidenceError( + "non-DECIMAL leaf carries precision or scale") + required = _required_converted(logical, bit_width, is_signed, time_unit) + if converted != required: + raise common.EvidenceError( + "logical type lacks its exact compatible converted type") + if type_length is not None and not INT32_MIN <= type_length <= INT32_MAX: + raise common.EvidenceError("type_length is outside signed Int32") + return { + "physical_type": physical, + "logical_type": logical, + "converted_type": converted, + "type_length": type_length, + "precision": precision, + "scale": scale, + "bit_width": bit_width, + "is_signed": is_signed, + "time_unit": time_unit, + "is_adjusted_to_utc": adjusted, + "crs": crs, + "geography_algorithm": algorithm, + } + + +def normalize_order(raw): + if raw is None: + return { + "state": "ABSENT", + "field_id": None, + "wire_type": None, + "header_hex": None, + } + states = { + "known": raw["member"], + "unknown": "UNKNOWN", + "wrong_type": "WRONG_TYPE", + "empty": "EMPTY", + } + state = states.get(raw["state"]) + if state not in ("TYPE_ORDER", "IEEE_754_TOTAL_ORDER", "UNKNOWN", + "WRONG_TYPE", "EMPTY"): + raise common.EvidenceError("raw ColumnOrder state is inconsistent") + return { + "state": state, + "field_id": raw["field_id"], + "wire_type": raw["wire_type"], + "header_hex": raw["header_hex"], + } + + +def _load_json(payload, label): + try: + return json.loads(payload.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as error: + raise common.EvidenceError(f"invalid JSON input: {label}: {error}") from error + + +def parse_corpus_manifest(payload): + if not isinstance(payload, bytes) or not payload: + raise common.EvidenceError("corpus manifest is empty or not bytes") + lines = payload.splitlines(keepends=True) + if len(lines) > 64: + raise common.EvidenceError("corpus manifest exceeds its record limit") + result = {} + order = [] + for line in lines: + match = CORPUS_LINE.fullmatch(line) + if match is None: + raise common.EvidenceError("corpus manifest row is not canonical") + digest = match.group(1).decode("ascii") + path = match.group(2).decode("ascii") + if path in result: + raise common.EvidenceError(f"duplicate corpus path: {path}") + result[path] = digest + order.append(path) + if order != sorted(order): + raise common.EvidenceError("corpus manifest paths are not sorted") + return result + + +def load_context(): + root = common.repository_root() + n6 = common.n6_root() + paths = { + "manifest": n6 / "manifest.toml", + "capabilities": n6 / "capabilities.toml", + "fixtures": n6 / "fixtures.toml", + "corpus": n6 / "corpus-files.sha256", + "schema": n6 / "evidence.schema.json", + "raw_schema": n6 / "oracles" / "raw-java" / "evidence.schema.json", + } + payloads = {"manifest": common.read_file_bytes(paths["manifest"], + 2 * 1024 * 1024)} + manifest = common.parse_toml_bytes(payloads["manifest"], paths["manifest"]) + common.verify_python_toolchain(manifest) + plan = root / manifest["plan_file"] + paths["plan"] = plan + for name, maximum in ( + ("capabilities", 2 * 1024 * 1024), + ("fixtures", 2 * 1024 * 1024), + ("corpus", 64 * 1024), + ("schema", 2 * 1024 * 1024), + ("raw_schema", 2 * 1024 * 1024), + ("plan", 2 * 1024 * 1024)): + payloads[name] = common.read_file_bytes(paths[name], maximum) + expected_hashes = { + "capabilities": manifest["capabilities_sha256"], + "fixtures": manifest["fixture_manifest_sha256"], + "corpus": manifest["corpus_manifest_sha256"], + "schema": manifest["evidence_schema_sha256"], + "plan": manifest["plan_sha256"], + } + for name, expected in expected_hashes.items(): + if common.sha256_bytes(payloads[name]) != expected: + raise common.EvidenceError( + f"{name} hash does not match the manifest") + capabilities = common.parse_toml_bytes( + payloads["capabilities"], paths["capabilities"]) + fixtures = common.parse_toml_bytes(payloads["fixtures"], paths["fixtures"]) + if capabilities.get("plan_sha256") != manifest["plan_sha256"] or \ + capabilities.get("fixture_manifest") != paths["fixtures"].name or \ + capabilities.get("unsupported_is_pass") is not False: + raise common.EvidenceError("capability matrix boundary is inconsistent") + if fixtures.get("checksum_manifest") != paths["corpus"].name: + raise common.EvidenceError("fixture corpus boundary is inconsistent") + corpus = parse_corpus_manifest(payloads["corpus"]) + apache = [item for item in fixtures["fixture"] + if item["source_kind"] == "apache-corpus"] + if any(item["status"] != "verified" or \ + item["authority"] != fixtures["authority"] or \ + item["source_revision"] != fixtures["source_revision"] + for item in apache): + raise common.EvidenceError("Apache corpus fixture authority is inconsistent") + fixture_corpus = {item["file"]: item["sha256"] for item in apache} + if len(fixture_corpus) != len(apache) or corpus != fixture_corpus: + raise common.EvidenceError("fixture files differ from the corpus manifest") + raw_entries = [item for item in manifest["frozen_evidence"] + if item["id"] == RAW_EVIDENCE_ID] + if len(raw_entries) != 1: + raise common.EvidenceError("raw evidence manifest entry is ambiguous") + raw_entry = raw_entries[0] + if raw_entry["status"] != "verified" or \ + raw_entry.get("storage") != "gate-generated": + raise common.EvidenceError("raw evidence identity is not verified") + if raw_entry["schema_file"] != \ + "test/conformance/n6/oracles/raw-java/evidence.schema.json": + raise common.EvidenceError("raw evidence schema path is unexpected") + if common.sha256_bytes(payloads["raw_schema"]) != raw_entry["schema_sha256"]: + raise common.EvidenceError("raw schema hash does not match the manifest") + raw_schema = _load_json(payloads["raw_schema"], paths["raw_schema"]) + normalized_schema = _load_json(payloads["schema"], paths["schema"]) + raw_class = jsonschema.validators.validator_for(raw_schema) + raw_class.check_schema(raw_schema) + normalized_class = jsonschema.validators.validator_for(normalized_schema) + normalized_class.check_schema(normalized_schema) + return { + "root": root, + "paths": paths, + "manifest": manifest, + "capabilities": capabilities, + "fixtures": fixtures, + "raw_entry": raw_entry, + "snapshots": {paths[name]: common.sha256_bytes(payload) + for name, payload in payloads.items()}, + "raw_validator": raw_class(raw_schema), + "normalized_validator": normalized_class(normalized_schema), + } + + +def _authority(context): + matches = [item for item in context["capabilities"]["authority"] + if item["id"] == PRODUCER] + if len(matches) != 1: + raise common.EvidenceError("raw authority is ambiguous") + authority = matches[0] + entry = context["raw_entry"] + if authority["version"] != "parquet-2.13-raw-footer-v3" or \ + authority["revision"] != FORMAT_COMMIT or \ + entry["authority"] != PRODUCER or \ + authority["toolchain_sha256"] != [entry["toolchain_sha256"]]: + raise common.EvidenceError("raw authority identity is inconsistent") + return authority + + +def fixture_cases(fixtures, fixture_set): + if fixture_set == "apache": + return [item for item in fixtures["fixture"] + if item["source_kind"] == "apache-corpus"] + if fixture_set != "generated": + raise common.EvidenceError(f"unknown fixture set: {fixture_set}") + cases = [] + for source in fixtures["generated_case"]: + if source["status"] != "verified" or \ + source["output_identity_status"] != "verified": + raise common.EvidenceError( + f"generated fixture identity is not verified: {source['id']}") + case = dict(source) + case["file"] = source["output_file"] + case["sha256"] = source["output_sha256"] + case["size"] = source["output_size"] + cases.append(case) + return cases + + +def _fixture_maps(fixtures, fixture_set): + cases = fixture_cases(fixtures, fixture_set) + by_name = {} + ids = set() + for case in cases: + if case["id"] in ids: + raise common.EvidenceError(f"duplicate fixture ID: {case['id']}") + ids.add(case["id"]) + name = pathlib.PurePosixPath(case["file"]).name + if name in by_name: + raise common.EvidenceError(f"duplicate fixture basename: {name}") + by_name[name] = case + return cases, by_name + + +def validate_normalized_payload(payload, context, upstream_payloads=()): + validator = context["root"] / "test" / "conformance" / "n6" / \ + "validate_evidence.py" + temporaries = [] + for value in (*upstream_payloads, payload): + with tempfile.NamedTemporaryFile(prefix="parquet-n6-normalized-", + suffix=".jsonl", delete=False) as stream: + temporary = pathlib.Path(stream.name) + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + temporaries.append(temporary) + try: + arguments = [ + sys.executable, + "-B", + "-I", + str(validator), + "--schema", str(context["paths"]["schema"]), + "--manifest", str(context["paths"]["manifest"]), + "--capabilities", str(context["paths"]["capabilities"]), + "--fixtures", str(context["paths"]["fixtures"]), + *(str(temporary) for temporary in temporaries), + ] + process = subprocess.run(arguments, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=60, check=False) + if len(process.stdout) > 64 * 1024 or len(process.stderr) > 64 * 1024: + raise common.EvidenceError("normalized validator output exceeds its limit") + if process.returncode != 0: + detail = process.stderr.decode("utf-8", errors="replace") + raise common.EvidenceError( + f"normalized output failed semantic validation: {detail}") + except (OSError, subprocess.TimeoutExpired) as error: + raise common.EvidenceError( + f"normalized validator could not run: {error}") from error + finally: + for temporary in temporaries: + temporary.unlink(missing_ok=True) + return None + + +def _check_leaf(raw, ordinal): + if raw["ordinal"] != ordinal: + raise common.EvidenceError("schema leaf ordinals are not contiguous") + if not raw["path"]: + raise common.EvidenceError("schema leaf path is empty") + return None + + +def _orders(metadata, leaves): + raw = metadata["column_orders"] + if not raw["present"]: + return [None for _ in leaves] + if raw["count"] != len(leaves) or len(raw["values"]) != len(leaves): + raise common.EvidenceError("ColumnOrder vector is not leaf aligned") + orders = [] + for ordinal, (order, leaf) in enumerate(zip(raw["values"], leaves)): + if order["ordinal"] != ordinal or order["schema_leaf_ordinal"] != ordinal: + raise common.EvidenceError("ColumnOrder ordinals are not leaf aligned") + if order["path"] != leaf["path"] or \ + order["physical_type"] != leaf["physical_type"] or \ + order["logical_type"] != leaf["logical_type"]["member"]: + raise common.EvidenceError("ColumnOrder schema facts are inconsistent") + orders.append(order) + return orders + + +def normalize_statistics(raw): + if len(raw["fields"]) != len(STATISTIC_FIELDS): + raise common.EvidenceError("Statistics field count is inconsistent") + output = { + "has_statistics": raw["present"], + "deprecated_min_hex": None, + "deprecated_max_hex": None, + "min_value_hex": None, + "max_value_hex": None, + "is_min_value_exact": None, + "is_max_value_exact": None, + "null_count": None, + "distinct_count": None, + "nan_count": None, + "unknown_statistics_field_ids": [], + } + for field, expected in zip(raw["fields"], STATISTIC_FIELDS): + field_id, name, target = expected + if field["field_id"] != field_id or field["name"] != name: + raise common.EvidenceError("Statistics field order is inconsistent") + value = field["value"] + if field["present"] and isinstance(value, dict): + if value["byte_length"] * 2 != len(value["hex"]): + raise common.EvidenceError("binary Statistics length is inconsistent") + value = value["hex"] + if field["present"] and target in ( + "null_count", "distinct_count", "nan_count"): + if not isinstance(value, int) or isinstance(value, bool) or \ + not -(1 << 63) <= value <= INT64_MAX: + raise common.EvidenceError( + f"{target} is outside signed Int64") + value = str(value) + output[target] = value if field["present"] else None + if not raw["present"] and any(value is not None + for key, value in output.items() + if key not in ("has_statistics", "unknown_statistics_field_ids")): + raise common.EvidenceError("absent Statistics carries a value") + return output + + +def _column_record(case, leaves, orders, row_group, column): + ordinal = column["schema_leaf_ordinal"] + if not column["metadata_present"] or not isinstance(ordinal, int) or \ + not 0 <= ordinal < len(leaves): + raise common.EvidenceError("column metadata does not map to a schema leaf") + leaf = leaves[ordinal] + if column["ordinal"] != ordinal or column["path"] != leaf["path"] or \ + column["physical_type"] != leaf["physical_type"] or \ + column["logical_type"] != leaf["logical_type"]["member"]: + raise common.EvidenceError("column and schema leaf facts are inconsistent") + num_values = column["num_values"] + if not isinstance(num_values, int) or isinstance(num_values, bool) or \ + not 0 <= num_values <= INT64_MAX: + raise common.EvidenceError("column num_values is outside signed Int64") + record = { + "record": "column_statistics", + "schema_version": 2, + "case_id": case["id"], + "file": case["file"], + "row_group": row_group["ordinal"], + "leaf": ordinal, + "path": leaf["path"], + "leaf_schema": effective_leaf(leaf), + "column_order": normalize_order(orders[ordinal]), + "num_values": str(num_values), + } + record.update(normalize_statistics(column["statistics"])) + return record + + +def _normalize_case(case, raw): + if raw["file"] != pathlib.PurePosixPath(case["file"]).name or \ + raw["file_sha256"] != case["sha256"] or \ + raw["file_size"] != case["size"]: + raise common.EvidenceError(f"raw file identity mismatch: {case['id']}") + metadata = raw["file_metadata"] + leaves = metadata["schema_leaves"] + if metadata["schema_leaf_count"] != len(leaves) or \ + len(leaves) != case["leaf_count"]: + raise common.EvidenceError(f"raw leaf count mismatch: {case['id']}") + for ordinal, leaf in enumerate(leaves): + _check_leaf(leaf, ordinal) + if len({tuple(leaf["path"]) for leaf in leaves}) != len(leaves): + raise common.EvidenceError(f"raw leaf paths are not unique: {case['id']}") + row_groups = metadata["row_groups"] + if metadata["row_group_count"] != len(row_groups) or \ + len(row_groups) != case["row_group_count"]: + raise common.EvidenceError(f"raw row-group count mismatch: {case['id']}") + if metadata["num_rows"] != sum(group["num_rows"] for group in row_groups): + raise common.EvidenceError(f"raw row counts are inconsistent: {case['id']}") + orders = _orders(metadata, leaves) + file_record = { + "record": "file", + "schema_version": 2, + "case_id": case["id"], + "file": case["file"], + "sha256": case["sha256"], + "size": case["size"], + "footer_length": raw["footer_length"], + "row_group_count": case["row_group_count"], + "leaf_count": case["leaf_count"], + "column_order_count": len(orders) + if metadata["column_orders"]["present"] else None, + "created_by_present": metadata["created_by"]["present"], + "created_by": metadata["created_by"]["value"], + } + columns = [] + for ordinal, row_group in enumerate(row_groups): + if row_group["ordinal"] != ordinal or \ + row_group["column_count"] != len(row_group["columns"]) or \ + len(row_group["columns"]) != len(leaves): + raise common.EvidenceError( + f"raw row-group topology mismatch: {case['id']}") + mapped = [_column_record(case, leaves, orders, + row_group, column) for column in row_group["columns"]] + if sorted(record["leaf"] for record in mapped) != list(range(len(leaves))): + raise common.EvidenceError( + f"raw row-group leaves are incomplete: {case['id']}") + columns.extend(sorted(mapped, key=lambda record: record["leaf"])) + expected = 1 + len(columns) + if case["normalized_record_count"] != expected: + raise common.EvidenceError( + f"normalized topology count mismatch: {case['id']}") + return file_record, columns + + +def _run_record(context, evidence_id, unsupported_cases): + authority = _authority(context) + if len(authority["toolchain_sha256"]) != 1: + raise common.EvidenceError("raw authority must bind one toolchain") + manifest = context["manifest"] + paths = context["paths"] + return { + "record": "run", + "schema_version": 2, + "evidence_id": evidence_id, + "producer": PRODUCER, + "producer_version": authority["version"], + "source_revision": authority["revision"], + "plan_sha256": manifest["plan_sha256"], + "capabilities_sha256": context["snapshots"][paths["capabilities"]], + "fixture_manifest_sha256": context["snapshots"][paths["fixtures"]], + "corpus_manifest_sha256": context["snapshots"][paths["corpus"]], + "evidence_schema_sha256": context["snapshots"][paths["schema"]], + "toolchain_sha256": authority["toolchain_sha256"][0], + "unsupported_cases": unsupported_cases, + } + + +def _claims(context): + authority = _authority(context) + claims = {} + for claim in authority.get("claim", []): + if claim["capability"] not in WIRE_CAPABILITIES or \ + claim["status"] not in ("verified", "planned"): + continue + for case_id in claim["cases"]: + key = (case_id, claim["capability"]) + if key in claims: + raise common.EvidenceError(f"duplicate raw claim: {key}") + claims[key] = claim["status"] + return claims + + +def _unsupported_claims(context): + claims = [] + seen = set() + for claim in _authority(context).get("claim", []): + if claim["status"] != "unsupported": + continue + for case_id in claim["cases"]: + key = (case_id, claim["capability"]) + if key in seen: + raise common.EvidenceError( + f"duplicate unsupported raw claim: {key}") + seen.add(key) + claims.append(key) + return sorted(claims) + + +def _result(case, capability, file_record, columns): + observations = [{ + "file": file_record, + "columns": columns, + }] + digest = common.capability_digest(case["id"], capability, observations) + return { + "record": "case_result", + "schema_version": 2, + "case_id": case["id"], + "capability_id": capability, + "digest_contract": case.get( + "digest_contract", "n6-capability-result-sha256-v1"), + "status": "PASS", + "expected_sha256": digest, + "actual_sha256": digest, + "detail": "Exact raw Parquet 2.13 wire facts normalized without semantic interpretation.", + } + + +def _unsupported_result(case_id, capability): + return { + "record": "case_result", + "schema_version": 2, + "case_id": case_id, + "capability_id": capability, + "digest_contract": "n6-capability-result-sha256-v1", + "status": "UNSUPPORTED", + "expected_sha256": None, + "actual_sha256": None, + "detail": "The reviewed capability matrix marks this result unsupported.", + } + + +def _generated_raw_entry(context): + matches = [item for item in context["manifest"]["frozen_evidence"] + if item["id"] == GENERATED_RAW_EVIDENCE_ID] + if len(matches) > 1: + raise common.EvidenceError("generated raw evidence entry is ambiguous") + if not matches: + return { + "id": GENERATED_RAW_EVIDENCE_ID, + "status": "verified", + "authority": PRODUCER, + "storage": "gate-generated", + "schema_file": + "test/conformance/n6/oracles/raw-java/evidence.schema.json", + "schema_sha256": context["raw_entry"]["schema_sha256"], + "toolchain_sha256": context["raw_entry"]["toolchain_sha256"], + "case_count": GENERATED_RAW_CASE_COUNT, + "record_count": GENERATED_RAW_RECORD_COUNT, + "sha256": GENERATED_RAW_SHA256, + } + entry = matches[0] + expected = { + "status": "verified", + "authority": PRODUCER, + "storage": "gate-generated", + "schema_file": + "test/conformance/n6/oracles/raw-java/evidence.schema.json", + "schema_sha256": context["raw_entry"]["schema_sha256"], + "toolchain_sha256": context["raw_entry"]["toolchain_sha256"], + "case_count": GENERATED_RAW_CASE_COUNT, + "record_count": GENERATED_RAW_RECORD_COUNT, + "sha256": GENERATED_RAW_SHA256, + } + if any(entry.get(field) != value for field, value in expected.items()): + raise common.EvidenceError("generated raw evidence identity is inconsistent") + return entry + + +def _profile(context, fixture_set): + if fixture_set == "apache": + return context["raw_entry"], EVIDENCE_ID + if fixture_set == "generated": + return _generated_raw_entry(context), GENERATED_EVIDENCE_ID + raise common.EvidenceError(f"unknown fixture set: {fixture_set}") + + +def render_raw_evidence(raw_path, fixture_set="apache"): + context = load_context() + entry, evidence_id = _profile(context, fixture_set) + limits = context["manifest"]["evidence_limits"] + raw_payload = common.read_file_bytes(raw_path, limits["max_file_bytes"]) + if common.sha256_bytes(raw_payload) != entry["sha256"]: + raise common.EvidenceError("raw evidence hash does not match its frozen identity") + records = common.parse_jsonl_bytes(raw_payload, raw_path, + limits["max_line_bytes"], entry["record_count"]) + if len(records) != entry["record_count"]: + raise common.EvidenceError("raw evidence record count is incomplete") + by_file = {} + for record in records: + context["raw_validator"].validate(record) + if record["evidence_version"] != "parquet-2.13-raw-footer-v3" or \ + record["format_commit"] != FORMAT_COMMIT or \ + record["thrift_version"] != "0.23.0": + raise common.EvidenceError("raw evidence version is unexpected") + if record["file"] in by_file: + raise common.EvidenceError(f"duplicate raw file: {record['file']}") + by_file[record["file"]] = record + cases, fixture_by_name = _fixture_maps(context["fixtures"], fixture_set) + if set(by_file) != set(fixture_by_name): + raise common.EvidenceError("raw evidence file set differs from the fixture corpus") + if len(cases) != entry["case_count"]: + raise common.EvidenceError("raw evidence case count differs from its identity") + claims = _claims(context) + unsupported = _unsupported_claims(context) if fixture_set == "apache" else [] + unsupported_cases = sorted({case_id for case_id, _ in unsupported}) + output = [_run_record(context, evidence_id, unsupported_cases)] + for case in cases: + raw = by_file[pathlib.PurePosixPath(case["file"]).name] + file_record, columns = _normalize_case(case, raw) + output.append(file_record) + output.extend(columns) + capabilities = sorted(capability for capability in case["capabilities"] + if (case["id"], capability) in claims) + output.extend(_result(case, capability, file_record, columns) + for capability in capabilities) + output.extend(_unsupported_result(case_id, capability) + for case_id, capability in unsupported) + for record in output: + context["normalized_validator"].validate(record) + payload = common.jsonl_bytes(output) + if len(payload) > limits["max_file_bytes"]: + raise common.EvidenceError("normalized raw evidence exceeds its byte limit") + if len(output) > limits["max_records_per_input"]: + raise common.EvidenceError("normalized raw evidence exceeds its record limit") + validate_normalized_payload(payload, context) + return payload + + +def parse_arguments(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--fixture-set", choices=("apache", "generated"), + default="apache") + parser.add_argument("--check", action="store_true") + return parser.parse_args(argv) + + +def main(argv=None): + try: + args = parse_arguments(argv) + payload = render_raw_evidence(args.input, args.fixture_set) + context = load_context() + common.ensure_snapshots(context["snapshots"]) + sources = (pathlib.Path(__file__), pathlib.Path(common.__file__)) + inputs = (args.input, *sources, *context["paths"].values()) + common.publish_bytes(args.output, payload, args.check, inputs) + action = "is fresh" if args.check else "written" + print(f"N6 {args.fixture_set} normalized raw evidence {action}: " + f"{args.output}") + return 0 + except (common.EvidenceError, OSError, ValueError, + jsonschema.ValidationError) as error: + print(f"N6 raw normalization failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/conformance/n6/normalizers/runtests.py b/test/conformance/n6/normalizers/runtests.py new file mode 100644 index 0000000..02b921d --- /dev/null +++ b/test/conformance/n6/normalizers/runtests.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +import contextlib +import copy +import io +import json +import os +import pathlib +import sys +import tempfile +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +import common +import normalize_model +import normalize_raw + + +class BoundedJsonlTests(unittest.TestCase): + def test_rejects_duplicate_keys_and_noncanonical_numbers(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + duplicate = root / "duplicate.jsonl" + duplicate.write_bytes(b'{"a":1,"a":2}\n') + with self.assertRaises(common.EvidenceError): + common.read_jsonl(duplicate, 100, 100, 1) + floating = root / "floating.jsonl" + floating.write_bytes(b'{"a":1.0}\n') + with self.assertRaises(common.EvidenceError): + common.read_jsonl(floating, 100, 100, 1) + + def test_rejects_missing_newline_and_every_limit(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path = root / "input.jsonl" + path.write_bytes(b"{}") + with self.assertRaises(common.EvidenceError): + common.read_jsonl(path, 2, 2, 1) + path.write_bytes(b"{}\n") + with self.assertRaises(common.EvidenceError): + common.read_jsonl(path, 2, 3, 1) + with self.assertRaises(common.EvidenceError): + common.read_jsonl(path, 3, 2, 1) + path.write_bytes(b"{}\n{}\n") + with self.assertRaises(common.EvidenceError): + common.read_jsonl(path, 6, 3, 1) + + def test_rejects_symbolic_link_input(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + target = root / "target.jsonl" + target.write_bytes(b"{}\n") + link = root / "link.jsonl" + link.symlink_to(target) + with self.assertRaises(common.EvidenceError): + common.read_jsonl(link, 3, 3, 1) + + def test_canonical_output_has_sorted_keys_and_no_floating_values(self): + self.assertEqual( + common.canonical_json({"z": 1, "a": [True, None]}), + '{"a":[true,null],"z":1}', + ) + with self.assertRaises(common.EvidenceError): + common.canonical_json({"value": 1.0}) + + +class LogicalNormalizationTests(unittest.TestCase): + def leaf(self): + return { + "physical_type": "INT32", + "type_length": {"present": False, "value": None}, + "converted_type": {"present": False, "value": None}, + "scale": {"present": False, "value": None}, + "precision": {"present": False, "value": None}, + "logical_type": { + "present": False, + "member": None, + "parameters": None, + }, + } + + def test_synthesizes_legacy_decimal_and_temporal_parameters(self): + decimal = self.leaf() + decimal["converted_type"] = {"present": True, "value": "DECIMAL"} + decimal["scale"] = {"present": True, "value": 2} + decimal["precision"] = {"present": True, "value": 9} + normalized = normalize_raw.effective_leaf(decimal) + self.assertEqual(normalized["logical_type"], "DECIMAL") + self.assertEqual(normalized["precision"], 9) + self.assertEqual(normalized["scale"], 2) + temporal = self.leaf() + temporal["converted_type"] = { + "present": True, + "value": "TIME_MILLIS", + } + normalized = normalize_raw.effective_leaf(temporal) + self.assertEqual(normalized["logical_type"], "TIME") + self.assertEqual(normalized["time_unit"], "MILLIS") + self.assertIs(normalized["is_adjusted_to_utc"], True) + + def test_modern_parameters_win_but_legacy_pair_must_match(self): + temporal = self.leaf() + temporal["converted_type"] = { + "present": True, + "value": "TIME_MICROS", + } + temporal["physical_type"] = "INT64" + temporal["logical_type"] = { + "present": True, + "member": "TIME", + "parameters": { + "unit": "MICROS", + "is_adjusted_to_utc": False, + }, + } + normalized = normalize_raw.effective_leaf(temporal) + self.assertIs(normalized["is_adjusted_to_utc"], False) + temporal["converted_type"]["value"] = "TIME_MILLIS" + with self.assertRaises(common.EvidenceError): + normalize_raw.effective_leaf(temporal) + + def test_rejects_unknown_future_logical_member(self): + leaf = self.leaf() + leaf["logical_type"] = { + "present": True, + "member": None, + "parameters": None, + } + with self.assertRaises(common.EvidenceError): + normalize_raw.effective_leaf(leaf) + + +class CorpusManifestTests(unittest.TestCase): + def test_parses_canonical_rows_and_rejects_duplicates(self): + first = b"0" * 64 + b" data/a.parquet\n" + second = b"1" * 64 + b" data/b.parquet\n" + self.assertEqual(normalize_raw.parse_corpus_manifest(first + second), { + "data/a.parquet": "0" * 64, + "data/b.parquet": "1" * 64, + }) + with self.assertRaises(common.EvidenceError): + normalize_raw.parse_corpus_manifest(first + first) + with self.assertRaises(common.EvidenceError): + normalize_raw.parse_corpus_manifest( + b"0" * 64 + b" data/a.parquet\n") + + def test_projects_only_verified_generated_fixture_identities(self): + generated = { + "id": "generated-case", + "status": "verified", + "source_kind": "julia-writer-generated", + "output_file": "generated/case.parquet", + "output_identity_status": "verified", + "output_sha256": "a" * 64, + "output_size": 12, + "row_group_count": 1, + "leaf_count": 1, + "normalized_record_count": 2, + "capabilities": ["wire.statistics.counts"], + } + cases = normalize_raw.fixture_cases({ + "fixture": [], + "generated_case": [generated], + }, "generated") + self.assertEqual(cases[0]["file"], "generated/case.parquet") + self.assertEqual(cases[0]["sha256"], "a" * 64) + self.assertEqual(cases[0]["size"], 12) + planned = copy.deepcopy(generated) + planned["output_identity_status"] = "planned" + with self.assertRaises(common.EvidenceError): + normalize_raw.fixture_cases({ + "fixture": [], + "generated_case": [planned], + }, "generated") + + def test_apache_fixture_projection_remains_the_default(self): + fixture = { + "id": "apache-case", + "status": "verified", + "source_kind": "apache-corpus", + "file": "data/case.parquet", + "sha256": "b" * 64, + "size": 12, + } + cases = normalize_raw.fixture_cases({ + "fixture": [fixture], + "generated_case": [], + }, "apache") + self.assertEqual(cases, [fixture]) + with self.assertRaises(common.EvidenceError): + normalize_raw.fixture_cases({ + "fixture": [fixture], + "generated_case": [], + }, "unknown") + + def test_preserves_explicit_unsupported_authority_claims(self): + context = { + "capabilities": {"authority": [{ + "id": "n6-raw-java", + "version": "parquet-2.13-raw-footer-v3", + "revision": normalize_raw.FORMAT_COMMIT, + "toolchain_sha256": ["a" * 64], + "claim": [{ + "capability": "semantic.type-order", + "status": "unsupported", + "cases": ["atomic-bound-family"], + }], + }]}, + "raw_entry": { + "authority": "n6-raw-java", + "toolchain_sha256": "a" * 64, + }, + } + self.assertEqual(normalize_raw._unsupported_claims(context), [ + ("atomic-bound-family", "semantic.type-order"), + ]) + result = normalize_raw._unsupported_result( + "atomic-bound-family", "semantic.type-order") + self.assertEqual(result["status"], "UNSUPPORTED") + self.assertIsNone(result["expected_sha256"]) + self.assertIsNone(result["actual_sha256"]) + + +class ColumnOrderTests(unittest.TestCase): + def test_preserves_known_unknown_wrong_type_empty_and_absent(self): + cases = ( + (None, "ABSENT", None), + ({"state": "known", "field_id": 1, "wire_type": 12, + "header_hex": "1c", "member": "TYPE_ORDER"}, + "TYPE_ORDER", "1c"), + ({"state": "unknown", "field_id": 7, "wire_type": 12, + "header_hex": "7c", "member": None}, "UNKNOWN", "7c"), + ({"state": "wrong_type", "field_id": 2, "wire_type": 8, + "header_hex": "25", "member": None}, "WRONG_TYPE", "25"), + ({"state": "empty", "field_id": None, "wire_type": None, + "header_hex": "00", "member": None}, "EMPTY", "00"), + ) + for raw, state, header in cases: + with self.subTest(state=state): + normalized = normalize_raw.normalize_order(raw) + self.assertEqual(normalized["state"], state) + self.assertEqual(normalized["header_hex"], header) + + def test_statistics_counts_become_canonical_int64_strings(self): + fields = [] + names = ( + "max", "min", "null_count", "distinct_count", "max_value", + "min_value", "is_max_value_exact", "is_min_value_exact", + "nan_count", + ) + values = (None, None, 0, 3, None, None, None, None, 2) + for field_id, (name, value) in enumerate(zip(names, values), 1): + fields.append({ + "field_id": field_id, + "name": name, + "present": value is not None, + "value": value, + }) + normalized = normalize_raw.normalize_statistics({ + "present": True, + "fields": fields, + }) + self.assertEqual(normalized["null_count"], "0") + self.assertEqual(normalized["distinct_count"], "3") + self.assertEqual(normalized["nan_count"], "2") + + +class AtomicOutputTests(unittest.TestCase): + def test_check_mode_and_atomic_replace(self): + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "evidence.jsonl" + common.publish_bytes(path, b"first\n", False, ()) + self.assertEqual(path.read_bytes(), b"first\n") + common.publish_bytes(path, b"first\n", True, ()) + with self.assertRaises(common.EvidenceError): + common.publish_bytes(path, b"second\n", True, ()) + self.assertEqual(path.read_bytes(), b"first\n") + + def test_rejects_input_alias_and_output_symlink(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "source.jsonl" + source.write_bytes(b"source\n") + with self.assertRaises(common.EvidenceError): + common.publish_bytes(source, b"changed\n", False, (source,)) + target = root / "target.jsonl" + target.write_bytes(b"target\n") + link = root / "link.jsonl" + link.symlink_to(target) + with self.assertRaises(common.EvidenceError): + common.publish_bytes(link, b"changed\n", False, ()) + self.assertEqual(target.read_bytes(), b"target\n") + + def test_normalization_failure_preserves_existing_output(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + raw = root / "raw.jsonl" + raw.write_bytes(b'{"bad":true}\n') + output = root / "output.jsonl" + output.write_bytes(b"sentinel\n") + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + status = normalize_raw.main([ + "--input", str(raw), + "--output", str(output), + ]) + self.assertNotEqual(status, 0) + self.assertEqual(output.read_bytes(), b"sentinel\n") + + def test_model_failure_preserves_existing_output(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + normalized = root / "normalized.jsonl" + normalized.write_bytes(b'{"bad":true}\n') + raw = root / "raw.jsonl" + raw.write_bytes(b'{"bad":true}\n') + output = root / "output.jsonl" + output.write_bytes(b"sentinel\n") + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + status = normalize_model.main([ + "--input", str(normalized), + "--raw-input", str(raw), + "--output", str(output), + "--julia-executable", os.environ.get( + "PARQUET_N6_TEST_JULIA_EXECUTABLE", "/absent/julia"), + ]) + self.assertNotEqual(status, 0) + self.assertEqual(output.read_bytes(), b"sentinel\n") + + def test_semantic_validator_rejects_schema_only_payload(self): + context = normalize_raw.load_context() + with self.assertRaises(common.EvidenceError): + normalize_raw.validate_normalized_payload(b"{}\n", context) + + +class IndependentModelBridgeTests(unittest.TestCase): + def julia_executable(self): + executable = os.environ.get("PARQUET_N6_TEST_JULIA_EXECUTABLE", "") + if not executable: + self.fail("PARQUET_N6_TEST_JULIA_EXECUTABLE is required") + return executable + + def run_model(self, columns, files, command, run_suite=False): + context = normalize_raw.load_context() + normalize_model._verify_models(context) + with normalize_model.model_producer_snapshot(context) as producer_root: + return normalize_model.run_model(columns, files, command, + producer_root, run_suite=run_suite) + + def test_model_producer_descriptor_binds_all_execution_sources(self): + context = normalize_raw.load_context() + self.assertIsNone(normalize_model._verify_models(context)) + descriptor = normalize_model.PRODUCER_DESCRIPTOR_PATH + self.assertEqual( + context["model_snapshots"][descriptor], + context["manifest"]["model_producer_descriptor_sha256"], + ) + broken = dict(context) + broken["manifest"] = dict(context["manifest"]) + broken["manifest"]["model_producer_descriptor_sha256"] = "0" * 64 + with self.assertRaisesRegex(common.EvidenceError, "descriptor hash"): + normalize_model._verify_models(broken) + + def test_model_evidence_binds_normalized_raw_input(self): + context = normalize_raw.load_context() + upstream = normalize_model._raw_upstream(context, b"raw evidence\n") + self.assertEqual(upstream, [{ + "evidence_id": "normalized-raw-java-apache-corpus", + "file": "test/conformance/n6/evidence/" + "raw-java-apache-corpus.normalized.jsonl", + "sha256": common.sha256_bytes(b"raw evidence\n"), + }]) + + def sample(self): + file_record = { + "case_id": "bridge-test", + "leaf_count": 1, + "created_by_present": True, + "created_by": "parquet-mr version 1.10.0", + } + column = { + "case_id": "bridge-test", + "row_group": 0, + "leaf": 0, + "leaf_schema": { + "physical_type": "INT32", + "logical_type": "NONE", + "type_length": None, + "bit_width": None, + "is_signed": None, + "precision": None, + "time_unit": None, + }, + "column_order": {"state": "TYPE_ORDER"}, + "num_values": "2", + "deprecated_min_hex": None, + "deprecated_max_hex": None, + "min_value_hex": "ffffffff", + "max_value_hex": "02000000", + "is_min_value_exact": True, + "is_max_value_exact": True, + "null_count": "0", + "distinct_count": "2", + "nan_count": None, + } + return file_record, column + + def test_interprets_signed_bounds_with_exact_toolchain_identity(self): + file_record, column = self.sample() + toolchain, results = self.run_model( + [column], {"bridge-test": file_record}, [self.julia_executable()]) + self.assertIn(toolchain, (item["runtime_tree"] for item in + normalize_model.ALLOWED_JULIA_TOOLCHAINS.values())) + result = results[("bridge-test", 0, 0)] + self.assertEqual(result["outcome"], "OK") + self.assertEqual(result["lower"]["value"], { + "kind": "SIGNED", + "value": "-1", + }) + self.assertEqual(result["upper"]["value"], { + "kind": "SIGNED", + "value": "2", + }) + + def test_rejects_any_model_format_error_before_claims(self): + file_record, column = self.sample() + malformed = copy.deepcopy(column) + malformed["min_value_hex"] = "ff" + _, results = self.run_model( + [malformed], {"bridge-test": file_record}, + [self.julia_executable()]) + self.assertEqual( + results[("bridge-test", 0, 0)]["outcome"], "FORMAT_ERROR") + with self.assertRaises(common.EvidenceError): + normalize_model.require_model_success(results) + + def test_rejects_unpinned_model_command(self): + file_record, column = self.sample() + with self.assertRaisesRegex(common.EvidenceError, + "absolute and canonical"): + self.run_model( + [column], {"bridge-test": file_record}, ["julia"]) + + def test_runs_frozen_independent_model_suite(self): + file_record, column = self.sample() + toolchain, results = self.run_model( + [column], {"bridge-test": file_record}, [self.julia_executable()], + run_suite=True) + self.assertIn(toolchain, + (item["runtime_tree"] for item in + normalize_model.ALLOWED_JULIA_TOOLCHAINS.values())) + self.assertEqual(results[("bridge-test", 0, 0)]["outcome"], "OK") + + def test_preserves_ieee_and_float16_raw_bits(self): + file_record, ieee = self.sample() + file_record["leaf_count"] = 2 + ieee["leaf_schema"]["physical_type"] = "FLOAT" + ieee["column_order"]["state"] = "IEEE_754_TOTAL_ORDER" + ieee["min_value_hex"] = "00000080" + ieee["max_value_hex"] = "00000000" + ieee["nan_count"] = "0" + float16 = copy.deepcopy(ieee) + float16["leaf"] = 1 + float16["leaf_schema"].update({ + "physical_type": "FIXED_LEN_BYTE_ARRAY", + "logical_type": "FLOAT16", + "type_length": 2, + }) + float16["column_order"]["state"] = "TYPE_ORDER" + float16["min_value_hex"] = "00c0" + float16["max_value_hex"] = "0040" + float16["nan_count"] = None + _, results = self.run_model( + [ieee, float16], {"bridge-test": file_record}, + [self.julia_executable()]) + ieee_result = results[("bridge-test", 0, 0)] + self.assertEqual(ieee_result["comparator"], "COMPARATOR_IEEE_FLOAT") + self.assertEqual(ieee_result["lower"]["value"]["bits_hex"], "80000000") + self.assertEqual(ieee_result["upper"]["value"]["bits_hex"], "00000000") + half_result = results[("bridge-test", 0, 1)] + self.assertEqual(half_result["comparator"], "COMPARATOR_TYPE_FLOAT") + self.assertEqual(half_result["lower"]["value"]["bits_hex"], "c000") + self.assertEqual(half_result["upper"]["value"]["bits_hex"], "4000") + + def test_bridge_has_no_production_dependency(self): + bridge = normalize_model.BRIDGE_PATH.read_text(encoding="utf-8") + forbidden = ( + "using " + "Parquet", + "import " + "Parquet", + "src/" + "statistics.jl", + "src/" + "write_statistics.jl", + ) + for token in forbidden: + self.assertNotIn(token, bridge) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/conformance/n6/oracles/arrow-rs/.gitignore b/test/conformance/n6/oracles/arrow-rs/.gitignore new file mode 100644 index 0000000..362e476 --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +/build/ diff --git a/test/conformance/n6/oracles/arrow-rs/README.md b/test/conformance/n6/oracles/arrow-rs/README.md new file mode 100644 index 0000000..0f8868f --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/README.md @@ -0,0 +1,79 @@ +# N6 Arrow Rust interoperability harness + +This harness runs the exact Arrow Rust 59.2.0 oracle that is already present in +the local canonical N5 image. A small N6 metadata oracle adds the positive +`TYPE_ORDER` observation that the N5 binary does not expose. `build.sh` compiles +that source inside the same immutable image from its frozen offline Cargo lock +and vendored sources. The ignored `build/` cache is not an authority input. The +descriptor pins its exact binary bytes. No build or run uses the network. + +Each container has `--pull never`, `--network none`, a read-only root, a +read-only corpus mount, no capabilities, no new privileges, and fixed CPU, +memory, process, output, and time limits. The host wrapper rejects symbolic +links in all input and output paths. It writes evidence with a same-directory +temporary file, `fsync`, and atomic replacement. The only repository output it +accepts is the declared Arrow Rust evidence path. Output inside the corpus is +forbidden. + +The base logical output has 30 records. It covers 14 files, 13 passing +`read.logical-values` results under `n6-logical-values-v1`, and two explicit +`UNSUPPORTED` results for `wire.column-order.ieee` and +`wire.statistics.nan-count`. The full reviewed scope adds ten passing +`wire.column-order.type` results and five new file records. The total is then +45 records. The metadata oracle compares every exposed order, path, physical +type, and value count with the frozen raw facts before it emits the shared raw +observation envelope. The mixed IEEE fixture remains outside this positive +scope because Arrow Rust exposes its IEEE member as unknown. + +Use the explicit `--draft` option while the evidence entry is planned. Draft +mode does not relax any input, source, binary, or descriptor digest check. The +descriptor must have the exact hash authorized in `capabilities.toml`. A normal +run requires both the authorized descriptor and a verified evidence entry. + +Build the metadata oracle once before enabling the positive TYPE_ORDER claims: + +```sh +test/conformance/n6/oracles/arrow-rs/build.sh +``` + +Set `PARQUET_N6_INTEROP_PYTHON_ROOT` to the reviewed CPython 3.12.8 base tree. +The launcher derives only `$PARQUET_N6_INTEROP_PYTHON_ROOT/bin/python3.12` and +uses `-I -B -S`. The wrapper then verifies the executable, full base runtime +tree, version, platform, and isolation flags against `toolchain.toml`. + +Run and check a provisional output outside the repository: + +```sh +mkdir -p /private/tmp/parquet-n6-arrow-rs +export PARQUET_N6_INTEROP_PYTHON_ROOT=/path/to/reviewed-cpython-3.12.8 +test/conformance/n6/oracles/arrow-rs/run.sh \ + /private/tmp/parquet-testing-09f3cdb \ + test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl \ + /private/tmp/parquet-n6-arrow-rs/arrow-rs.normalized.jsonl \ + --draft +test/conformance/n6/oracles/arrow-rs/check.sh \ + /private/tmp/parquet-testing-09f3cdb \ + test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl \ + /private/tmp/parquet-n6-arrow-rs/arrow-rs.normalized.jsonl \ + --draft +``` + +Run unit tests and the two-pass container integration test: + +```sh +"$PARQUET_N6_INTEROP_PYTHON_ROOT/bin/python3.12" -I -B -S \ + test/conformance/n6/oracles/arrow-rs/runtests.py \ + --integration \ + --repository . \ + --corpus-root /private/tmp/parquet-testing-09f3cdb \ + --raw-evidence test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl \ + --draft +``` + +The integration command writes the exact manifest target at +`test/conformance/n6/evidence/arrow-rs.normalized.jsonl`, then checks it without +changing its bytes. It checks all shared logical digests, both unsupported +records, exact file coverage, deterministic write/check behavior, and the +record boundary selected by the exact authority scope. Checked-in normalized +evidence remains provisional until the capability and toolchain records receive +their final review and freeze. diff --git a/test/conformance/n6/oracles/arrow-rs/build.sh b/test/conformance/n6/oracles/arrow-rs/build.sh new file mode 100644 index 0000000..b9d507d --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/build.sh @@ -0,0 +1,76 @@ +#!/bin/sh +set -eu + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +source_dir="$oracle_dir/metadata" +build_root="$oracle_dir/build" +output="$build_root/parquet-jl-n6-arrow-rs-metadata" +docker=${PARQUET_N6_DOCKER:-docker} +image_reference=parquet-jl-n5-oracles:n5d-canonical-a +image_id=sha256:04e56f6512080165bff7c9ae869b27ab20b14fd2dbb7fd7dc3827bc2a1a6568b +expected_binary=a26a0a29f99adde346800e85ee66544226d2105a040b40eb3536e8d629d8c301 +expected_size=710064 + +if [ -L "$build_root" ] || \ + { [ -e "$build_root" ] && [ ! -d "$build_root" ]; }; then + echo "Arrow Rust build root must be a real directory" >&2 + exit 1 +fi +if [ -L "$output" ] || { [ -e "$output" ] && [ ! -f "$output" ]; }; then + echo "Arrow Rust metadata output must be a regular non-link file" >&2 + exit 1 +fi +for input in "$source_dir/Cargo.toml" "$source_dir/src/main.rs"; do + if [ -L "$input" ] || [ ! -f "$input" ]; then + echo "Arrow Rust metadata source must be a regular non-link file" >&2 + exit 1 + fi +done +case "$source_dir" in + *[,:]*) + echo "Arrow Rust metadata source path cannot be mounted safely" >&2 + exit 1 + ;; +esac +identity=$($docker image inspect --format \ + '{{.Id}}|{{.Architecture}}|{{.Os}}' "$image_reference") +if [ "$identity" != "$image_id|amd64|linux" ]; then + echo "local Arrow Rust image identity differs" >&2 + exit 1 +fi + +mkdir -p "$build_root" +if [ -L "$build_root" ] || [ ! -d "$build_root" ]; then + echo "Arrow Rust build root changed while it was created" >&2 + exit 1 +fi +temporary=$(mktemp -d "$build_root/.build.XXXXXX") +trap 'rm -rf "$temporary"' EXIT HUP INT TERM +user_id=$(id -u) +group_id=$(id -g) +$docker run --rm --pull never --network none --platform linux/amd64 \ + --read-only --cap-drop ALL --security-opt no-new-privileges \ + --pids-limit 128 --memory 2g --cpus 2 \ + --user "$user_id:$group_id" \ + --tmpfs /tmp:rw,nosuid,nodev,noexec,size=256m \ + --mount "type=bind,source=$source_dir,target=/source,readonly" \ + --mount "type=bind,source=$temporary,target=/build" \ + --workdir /build --entrypoint /bin/sh "$image_id" -c ' + mkdir -p /build/src + cp /source/Cargo.toml /build/Cargo.toml + cp /source/src/main.rs /build/src/main.rs + cp /opt/bootstrap/arrow-rs/Cargo.lock /build/Cargo.lock + CARGO_TARGET_DIR=/build/target cargo build --release --offline --locked + cp /build/target/release/parquet-jl-n5-arrow-rs-oracle \ + /build/parquet-jl-n6-arrow-rs-metadata + ' +candidate="$temporary/parquet-jl-n6-arrow-rs-metadata" +actual=$(shasum -a 256 "$candidate" | awk '{print $1}') +if [ "$actual" != "$expected_binary" ] || \ + [ "$(stat -f '%z' "$candidate")" != "$expected_size" ]; then + echo "Arrow Rust metadata binary identity differs" >&2 + exit 1 +fi +chmod 0555 "$candidate" +mv -f "$candidate" "$output" +printf 'Built the pinned Arrow Rust N6 metadata oracle.\n' diff --git a/test/conformance/n6/oracles/arrow-rs/check.sh b/test/conformance/n6/oracles/arrow-rs/check.sh new file mode 100755 index 0000000..40239fb --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/check.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +exec "$oracle_dir/run.sh" "$@" --check diff --git a/test/conformance/n6/oracles/arrow-rs/metadata/Cargo.toml b/test/conformance/n6/oracles/arrow-rs/metadata/Cargo.toml new file mode 100644 index 0000000..6e6a231 --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/metadata/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "parquet-jl-n5-arrow-rs-oracle" +version = "0.1.0" +edition = "2024" +rust-version = "1.96.1" +publish = false +license = "MIT" +description = "Offline Arrow Rust TYPE_ORDER metadata oracle for Parquet.jl N6" + +[dependencies] +arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +arrow-json = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" } +parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "782e5a685501a9db6cc8e9a3b7cbff894940c47a", default-features = true } +serde = { version = "=1.0.228", features = ["derive"] } +serde_json = "=1.0.145" +sha2 = "=0.10.9" + +[profile.release] +codegen-units = 1 +lto = "thin" +strip = "debuginfo" diff --git a/test/conformance/n6/oracles/arrow-rs/metadata/src/main.rs b/test/conformance/n6/oracles/arrow-rs/metadata/src/main.rs new file mode 100644 index 0000000..a41e19d --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/metadata/src/main.rs @@ -0,0 +1,144 @@ +use std::env; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; + +use parquet::basic::ColumnOrder; +use parquet::file::reader::{FileReader, SerializedFileReader}; +use serde::Serialize; + +type DynError = Box; +type Result = std::result::Result; + +const PRODUCER: &str = "arrow-rs"; +const PRODUCER_VERSION: &str = "59.2.0"; +const SOURCE_REVISION: &str = "782e5a685501a9db6cc8e9a3b7cbff894940c47a"; +const RUST_TOOLCHAIN: &str = "1.96.1"; +const RUSTC: &str = "rustc 1.96.1 (31fca3adb 2026-06-26)"; +const CARGO: &str = "cargo 1.96.1 (356927216 2026-06-26)"; + +#[derive(Serialize)] +struct ToolEvidence { + name: &'static str, + version: &'static str, + commit: &'static str, + rust_toolchain: &'static str, + rustc: &'static str, + cargo: &'static str, +} + +#[derive(Serialize)] +struct ColumnEvidence { + path: Vec, + physical_type: String, + column_order: &'static str, + num_values: i64, +} + +#[derive(Serialize)] +struct RowGroupEvidence { + row_group: usize, + row_count: i64, + columns: Vec, +} + +#[derive(Serialize)] +struct MetadataEvidence { + oracle: ToolEvidence, + action: &'static str, + file_name: String, + file_bytes: u64, + row_group_count: usize, + leaf_count: usize, + row_groups: Vec, +} + +fn checked_input(value: &str) -> Result { + let path = PathBuf::from(value); + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err("metadata input must be a regular non-link file".into()); + } + return Ok(path); +} + +fn column_order_name(order: ColumnOrder) -> &'static str { + return match order { + ColumnOrder::TYPE_DEFINED_ORDER(_) => "TYPE_ORDER", + ColumnOrder::UNDEFINED => "UNDEFINED", + ColumnOrder::UNKNOWN => "UNKNOWN", + }; +} + +fn inspect(path: &Path) -> Result { + let file_bytes = fs::metadata(path)?.len(); + let reader = SerializedFileReader::new(File::open(path)?)?; + let metadata = reader.metadata(); + let file_metadata = metadata.file_metadata(); + let leaf_count = file_metadata.schema_descr().num_columns(); + let mut row_groups = Vec::with_capacity(metadata.num_row_groups()); + for row_group in 0..metadata.num_row_groups() { + let group = metadata.row_group(row_group); + if group.num_columns() != leaf_count { + return Err("row-group leaf count differs from the schema".into()); + } + let mut columns = Vec::with_capacity(leaf_count); + for leaf in 0..leaf_count { + let column = group.column(leaf); + columns.push(ColumnEvidence { + path: column.column_path().parts().to_vec(), + physical_type: format!("{:?}", column.column_type()), + column_order: column_order_name(file_metadata.column_order(leaf)), + num_values: column.num_values(), + }); + } + row_groups.push(RowGroupEvidence { + row_group, + row_count: group.num_rows(), + columns, + }); + } + let file_name = path + .file_name() + .ok_or("metadata input has no file name")? + .to_str() + .ok_or("metadata input file name is not UTF-8")? + .to_owned(); + return Ok(MetadataEvidence { + oracle: ToolEvidence { + name: PRODUCER, + version: PRODUCER_VERSION, + commit: SOURCE_REVISION, + rust_toolchain: RUST_TOOLCHAIN, + rustc: RUSTC, + cargo: CARGO, + }, + action: "type-order", + file_name, + file_bytes, + row_group_count: row_groups.len(), + leaf_count, + row_groups, + }); +} + +fn run() -> Result<()> { + let arguments: Vec = env::args().skip(1).collect(); + if arguments.len() != 3 || arguments[0] != "type-order" || arguments[1] != "--input" { + return Err("usage: arrow-rs-metadata type-order --input FILE".into()); + } + let evidence = inspect(&checked_input(&arguments[2])?)?; + let stdout = io::stdout(); + let mut output = stdout.lock(); + serde_json::to_writer(&mut output, &evidence)?; + use std::io::Write; + output.write_all(b"\n")?; + return Ok(()); +} + +fn main() { + if let Err(error) = run() { + eprintln!("arrow-rs-metadata: {error}"); + std::process::exit(1); + } +} diff --git a/test/conformance/n6/oracles/arrow-rs/run.py b/test/conformance/n6/oracles/arrow-rs/run.py new file mode 100755 index 0000000..61f2f40 --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/run.py @@ -0,0 +1,985 @@ +#!/usr/bin/env python3 +import argparse +import datetime +import decimal +import json +import os +import pathlib +import re +import selectors +import stat +import subprocess +import sys +import tempfile +import time + +SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parent +HARNESS_DIRECTORY = SCRIPT_DIRECTORY.parents[1] / "harnesses" +sys.path.insert(0, str(HARNESS_DIRECTORY)) +from common import (HarnessError, atomic_output, build_context, case_result, + checked_file, decimal_unscaled, evidence_bytes, leaf_records, + regular_file_bytes, repository_input, reject_output_alias, run_record, + sha256_bytes, sha256_file, verify_python_runtime) +sys.path = [entry for entry in sys.path + if pathlib.Path(entry or ".").resolve() != HARNESS_DIRECTORY] + + +PRODUCER = "arrow-rs" +EVIDENCE_ID = "normalized-arrow-rs" +DESCRIPTOR_RELATIVE = "test/conformance/n6/oracles/arrow-rs/toolchain.toml" +IMAGE_STDOUT_LIMIT = 8 * 1024 * 1024 +IMAGE_STDERR_LIMIT = 1024 * 1024 +IMAGE_TIMEOUT_SECONDS = 45 +EXPECTED_LOGICAL_CASES = { + "apache-alltypes-dictionary", + "apache-alltypes-plain", + "apache-binary", + "apache-bson", + "apache-byte-array-decimal", + "apache-fixed-length-byte-array", + "apache-fixed-length-decimal", + "apache-fixed-length-decimal-legacy", + "apache-int32-decimal", + "apache-int32-with-null-pages", + "apache-int64-decimal", + "apache-json", + "apache-rle-boolean-encoding", +} +EXPECTED_TYPE_ORDER_CASES = { + "apache-binary", + "apache-binary-truncated-min-max", + "apache-bson", + "apache-fixed-length-byte-array", + "apache-float16-nonzeros-and-nans", + "apache-float16-zeros-and-nans", + "apache-int32-with-null-pages", + "apache-json", + "apache-nan-in-stats", + "apache-single-nan", +} +EXPECTED_UNSUPPORTED = { + ("apache-floating-orders-nan-count", "wire.column-order.ieee"), + ("apache-floating-orders-nan-count", "wire.statistics.nan-count"), +} +EXPECTED_IMAGE_SOURCES = { + "/opt/bootstrap/arrow-rs/.gitignore", + "/opt/bootstrap/arrow-rs/Cargo.lock", + "/opt/bootstrap/arrow-rs/Cargo.toml", + "/opt/bootstrap/arrow-rs/README.md", + "/opt/bootstrap/arrow-rs/UPSTREAM.toml", + "/opt/bootstrap/arrow-rs/rust-toolchain.toml", + "/opt/bootstrap/arrow-rs/scripts/build.sh", + "/opt/bootstrap/arrow-rs/scripts/run.sh", + "/opt/bootstrap/arrow-rs/scripts/test.sh", + "/opt/bootstrap/arrow-rs/src/cases.rs", + "/opt/bootstrap/arrow-rs/src/evidence.rs", + "/opt/bootstrap/arrow-rs/src/main.rs", +} +EXPECTED_WRAPPERS = { + "test/conformance/n6/harnesses/common.py", + "test/conformance/n6/oracles/arrow-rs/build.sh", + "test/conformance/n6/oracles/arrow-rs/check.sh", + "test/conformance/n6/oracles/arrow-rs/metadata/Cargo.toml", + "test/conformance/n6/oracles/arrow-rs/metadata/src/main.rs", + "test/conformance/n6/oracles/arrow-rs/run.py", + "test/conformance/n6/oracles/arrow-rs/run.sh", + "test/conformance/n6/oracles/arrow-rs/runtests.py", +} +TIMESTAMP_PATTERN = re.compile( + r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})" + r"(?:\.(\d{1,9}))?$") +HEX_PATTERN = re.compile(r"^[0-9a-f]*$") + + +def reject_symlink_components(path, label, allow_missing_leaf=False): + absolute = pathlib.Path(os.path.abspath(path)) + current = pathlib.Path(absolute.anchor) + parts = absolute.parts[1:] if absolute.anchor else absolute.parts + for index, part in enumerate(parts): + current = current / part + try: + metadata = current.lstat() + except FileNotFoundError: + if allow_missing_leaf and index == len(parts) - 1: + return absolute + raise HarnessError(f"{label} does not exist: {current}") + if stat.S_ISLNK(metadata.st_mode): + raise HarnessError(f"{label} contains a symbolic link: {current}") + return absolute + + +def checked_directory(path, label): + path = reject_symlink_components(path, label) + if not path.is_dir(): + raise HarnessError(f"{label} is not a directory") + return path.resolve(strict=True) + + +def checked_regular_file(path, label): + path = reject_symlink_components(path, label) + if not path.is_file(): + raise HarnessError(f"{label} is not a regular file") + return path.resolve(strict=True) + + +def checked_output_path(path): + requested = pathlib.Path(os.path.abspath(path)) + checked_directory(requested.parent, "evidence output parent") + return reject_symlink_components(requested, "evidence output", + allow_missing_leaf=True) + + +def canonical_integer(value): + if value != "0" and (value.startswith("0") or value.startswith("-0")): + raise HarnessError(f"noncanonical JSON integer: {value}") + return int(value) + + +def unique_object(pairs): + output = {} + for key, value in pairs: + if key in output: + raise HarnessError(f"duplicate JSON object key: {key}") + output[key] = value + return output + + +def invalid_number(value): + raise HarnessError(f"invalid JSON number: {value}") + + +def load_oracle_json(value): + try: + decoded = value.decode("utf-8") + result = json.loads(decoded, object_pairs_hook=unique_object, + parse_constant=invalid_number, parse_float=invalid_number, + parse_int=canonical_integer) + except (UnicodeError, json.JSONDecodeError) as error: + raise HarnessError("Arrow Rust audit returned invalid JSON") from error + if not isinstance(result, dict): + raise HarnessError("Arrow Rust audit did not return a JSON object") + return result + + +def load_arrow_json_row(value): + try: + result = json.loads(value, object_pairs_hook=unique_object, + parse_constant=invalid_number, parse_float=decimal.Decimal, + parse_int=canonical_integer) + except (UnicodeError, json.JSONDecodeError, decimal.InvalidOperation) as error: + raise HarnessError("Arrow Rust JSON row is invalid") from error + if not isinstance(result, dict): + raise HarnessError("Arrow Rust JSON row is not an object") + return result + + +def run_bounded(command, maximum_stdout=IMAGE_STDOUT_LIMIT, + maximum_stderr=IMAGE_STDERR_LIMIT, + timeout_seconds=IMAGE_TIMEOUT_SECONDS): + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ, "stdout") + selector.register(process.stderr, selectors.EVENT_READ, "stderr") + output = {"stdout": bytearray(), "stderr": bytearray()} + limits = {"stdout": maximum_stdout, "stderr": maximum_stderr} + deadline = time.monotonic() + timeout_seconds + try: + while selector.get_map(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise HarnessError("bounded command timed out") + events = selector.select(remaining) + if not events: + raise HarnessError("bounded command timed out") + for key, _ in events: + chunk = os.read(key.fileobj.fileno(), 65536) + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + output[name].extend(chunk) + if len(output[name]) > limits[name]: + raise HarnessError(f"bounded command {name} exceeds its limit") + status = process.wait(timeout=max(0.1, deadline - time.monotonic())) + except BaseException: + process.kill() + process.wait() + raise + finally: + selector.close() + if status != 0: + diagnostic = bytes(output["stderr"]).decode("utf-8", "replace").strip() + if len(diagnostic) > 1000: + diagnostic = diagnostic[:1000] + "..." + raise HarnessError( + f"bounded command failed with status {status}: {diagnostic}") + return bytes(output["stdout"]), bytes(output["stderr"]) + + +def docker_base(arguments, descriptor): + return [ + arguments.docker, + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--platform", + descriptor["image_platform"], + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "64", + "--memory", + "512m", + "--cpus", + "1", + "--user", + "65534:65534", + "--tmpfs", + "/tmp:rw,nosuid,nodev,noexec,size=16m", + ] + + +def docker_image_identity(arguments, descriptor): + command = [arguments.docker, "image", "inspect", "--format", + "{{.Id}}|{{.Architecture}}|{{.Os}}", descriptor["image_reference"]] + stdout, _ = run_bounded(command, 4096, IMAGE_STDERR_LIMIT, 15) + expected = (f"{descriptor['image_id']}|amd64|linux\n").encode("ascii") + if stdout != expected: + raise HarnessError("local Arrow Rust image identity differs") + return None + + +def verify_image_files(arguments, descriptor): + expected = {descriptor["binary_path"]: descriptor["binary_sha256"]} + for item in descriptor["source"]: + expected[item["path"]] = item["sha256"] + command = docker_base(arguments, descriptor) + command.extend(["--entrypoint", "/usr/bin/sha256sum", + descriptor["image_id"]]) + command.extend(sorted(expected)) + stdout, _ = run_bounded(command, 65536) + observed = {} + try: + for line in stdout.decode("ascii").splitlines(): + digest, path = line.split(" ", 1) + if path in observed: + raise HarnessError(f"duplicate image identity path: {path}") + observed[path] = digest + except (UnicodeError, ValueError) as error: + raise HarnessError("image identity output is malformed") from error + if observed != expected: + raise HarnessError("Arrow Rust image file identities differ") + return None + + +def validate_descriptor(repository, descriptor, authority_record): + required = { + "descriptor_version", "status", "producer", "producer_version", + "source_revision", "image_reference", "image_id", "image_platform", + "binary_path", "binary_sha256", "platform", "python_version", + "python_distribution_url", "python_distribution_sha256", + "python_tree_policy", "python_executable_sha256", + "python_tree_sha256", "rust_toolchain", + "rustc", "cargo", "metadata_binary_file", + "metadata_binary_sha256", "metadata_binary_size", "source", "wrapper", + } + if set(descriptor) != required or descriptor["descriptor_version"] != 1: + raise HarnessError("Arrow Rust toolchain descriptor has invalid keys") + expected = { + "producer": PRODUCER, + "producer_version": authority_record["version"], + "source_revision": authority_record["revision"], + "image_platform": "linux/amd64", + "platform": "macos-15-arm64", + "python_version": "3.12.8", + "rust_toolchain": "1.96.1", + "rustc": "rustc 1.96.1 (31fca3adb 2026-06-26)", + "cargo": "cargo 1.96.1 (356927216 2026-06-26)", + } + for field, value in expected.items(): + if descriptor[field] != value: + raise HarnessError(f"Arrow Rust descriptor has stale {field}") + if descriptor["status"] not in ("planned", "verified"): + raise HarnessError("Arrow Rust descriptor has invalid status") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", descriptor["image_id"]): + raise HarnessError("Arrow Rust descriptor image ID is invalid") + if not re.fullmatch(r"[0-9a-f]{64}", descriptor["binary_sha256"]): + raise HarnessError("Arrow Rust descriptor binary digest is invalid") + if not re.fullmatch(r"[0-9a-f]{64}", + descriptor["metadata_binary_sha256"]): + raise HarnessError( + "Arrow Rust descriptor metadata binary digest is invalid") + if not isinstance(descriptor["metadata_binary_size"], int) or \ + isinstance(descriptor["metadata_binary_size"], bool) or \ + descriptor["metadata_binary_size"] <= 0: + raise HarnessError( + "Arrow Rust descriptor metadata binary size is invalid") + if not re.fullmatch(r"[0-9a-f]{64}", + descriptor["python_executable_sha256"]): + raise HarnessError("Arrow Rust descriptor Python digest is invalid") + if not re.fullmatch(r"[0-9a-f]{64}", descriptor["python_tree_sha256"]): + raise HarnessError("Arrow Rust descriptor Python tree digest is invalid") + if not descriptor["binary_path"].startswith("/"): + raise HarnessError("Arrow Rust descriptor binary path is not absolute") + paths = set() + for group in (descriptor["source"], descriptor["wrapper"]): + if not isinstance(group, list) or not group: + raise HarnessError("Arrow Rust descriptor identity list is empty") + for item in group: + if set(item) != {"path", "sha256"} or item["path"] in paths: + raise HarnessError("Arrow Rust descriptor identity is invalid") + if not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]): + raise HarnessError("Arrow Rust descriptor digest is invalid") + paths.add(item["path"]) + for item in descriptor["source"]: + if not item["path"].startswith("/opt/bootstrap/arrow-rs/"): + raise HarnessError("Arrow Rust source path is outside the image pin") + if {item["path"] for item in descriptor["source"]} != \ + EXPECTED_IMAGE_SOURCES: + raise HarnessError("Arrow Rust image source coverage differs") + if {item["path"] for item in descriptor["wrapper"]} != EXPECTED_WRAPPERS: + raise HarnessError("Arrow Rust wrapper coverage differs") + for item in descriptor["wrapper"]: + candidate = repository_input(repository, item["path"]) + if sha256_file(candidate) != item["sha256"]: + raise HarnessError(f"Arrow Rust wrapper digest differs: {item['path']}") + metadata_binary = repository_input(repository, + descriptor["metadata_binary_file"]) + if metadata_binary.stat().st_size != descriptor["metadata_binary_size"] or \ + sha256_file(metadata_binary) != \ + descriptor["metadata_binary_sha256"]: + raise HarnessError("Arrow Rust metadata binary identity differs") + verify_python_runtime(descriptor) + return descriptor + + +def audit_command(arguments, descriptor, corpus_root, relative): + if any(character in str(corpus_root) for character in (",", ":", "\n", "\r")): + raise HarnessError("corpus root cannot be encoded as a Docker mount") + command = docker_base(arguments, descriptor) + command.extend([ + "--mount", + f"type=bind,source={corpus_root},target=/corpus,readonly", + "--entrypoint", + descriptor["binary_path"], + descriptor["image_id"], + "audit", + "--input", + "/corpus/" + relative, + ]) + return command + + +def metadata_command(arguments, descriptor, corpus_root, relative, + binary_name): + if any(character in str(corpus_root) + for character in (",", ":", "\n", "\r")): + raise HarnessError("corpus root cannot be encoded as a Docker mount") + if "/" in binary_name or binary_name in ("", ".", ".."): + raise HarnessError("Arrow Rust metadata binary name is invalid") + command = docker_base(arguments, descriptor) + command.extend([ + "--mount", + f"type=bind,source={corpus_root},target=/corpus,readonly", + "--entrypoint", + "/corpus/" + binary_name, + descriptor["image_id"], + "type-order", + "--input", + "/corpus/" + relative, + ]) + return command + + +def snapshot_metadata_binary(repository, descriptor, root): + source = repository_input(repository, descriptor["metadata_binary_file"]) + value = regular_file_bytes(source, descriptor["metadata_binary_size"], + "Arrow Rust metadata binary") + if len(value) != descriptor["metadata_binary_size"] or \ + sha256_bytes(value) != descriptor["metadata_binary_sha256"]: + raise HarnessError("Arrow Rust metadata binary identity differs") + destination = pathlib.Path(root) / ".arrow-rs-metadata" + output = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o500) + try: + with os.fdopen(output, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.unlink(destination) + except FileNotFoundError: + pass + raise + return destination + + +def validate_tool_evidence(oracle, descriptor): + expected = { + "name": PRODUCER, + "version": descriptor["producer_version"], + "commit": descriptor["source_revision"], + "rust_toolchain": descriptor["rust_toolchain"], + "rustc": descriptor["rustc"], + "cargo": descriptor["cargo"], + } + if oracle != expected: + raise HarnessError("Arrow Rust audit tool identity differs") + return None + + +def expected_arrow_type(leaf): + schema = leaf["leaf_schema"] + physical = schema["physical_type"] + logical = schema["logical_type"] + if logical == "DECIMAL": + return f"Decimal128({schema['precision']}, {schema['scale']})" + if logical == "JSON": + return "Utf8" + if logical == "FLOAT16": + return "Float16" + if physical == "BOOLEAN": + return "Boolean" + if physical == "INT32": + return "Int32" + if physical == "INT64": + return "Int64" + if physical == "FLOAT": + return "Float32" + if physical == "DOUBLE": + return "Float64" + if physical == "INT96": + return "Timestamp(Nanosecond, None)" + if physical == "BYTE_ARRAY": + return "Binary" + if physical == "FIXED_LEN_BYTE_ARRAY": + return f"FixedSizeBinary({schema['type_length']})" + raise HarnessError(f"unsupported Arrow Rust physical type: {physical}") + + +def validate_audit_columns(evidence, leaves, fixture, raw_columns): + columns = evidence.get("columns") + if not isinstance(columns, list) or len(columns) != len(leaves): + raise HarnessError(f"Arrow Rust leaf count differs: {fixture['id']}") + row_group_rows = None + for leaf_ordinal, (leaf, column) in enumerate(zip(leaves, columns)): + required_column = { + "path", "physical_type", "maximum_definition_level", + "maximum_repetition_level", "row_groups", + } + maximum_definition = column.get("maximum_definition_level") + if set(column) != required_column or \ + not isinstance(maximum_definition, int) or \ + isinstance(maximum_definition, bool) or maximum_definition < 0 or \ + column.get("path") != leaf["path"] or \ + column.get("physical_type") != \ + leaf["leaf_schema"]["physical_type"]: + raise HarnessError(f"Arrow Rust leaf identity differs: {fixture['id']}") + if column.get("maximum_repetition_level") != 0: + raise HarnessError(f"Arrow Rust claimed fixture is nested: {fixture['id']}") + groups = column.get("row_groups") + if not isinstance(groups, list) or \ + len(groups) != fixture["row_group_count"]: + raise HarnessError(f"Arrow Rust row-group coverage differs: {fixture['id']}") + current_rows = [] + for ordinal, group in enumerate(groups): + rows = group.get("rows") + required_group = { + "row_group", "rows", "compression", "repetition", + "definition", "dense_values", "pages", + } + raw = raw_columns.get((fixture["id"], ordinal, leaf_ordinal)) + if set(group) != required_group or raw is None or \ + rows != int(raw["num_values"]) or \ + group.get("row_group") != ordinal or \ + not isinstance(rows, int) or isinstance(rows, bool) or rows < 0: + raise HarnessError(f"Arrow Rust row-group identity differs: {fixture['id']}") + repetition = group.get("repetition") + definition = group.get("definition") + dense = group.get("dense_values") + if not all(isinstance(value, list) + for value in (repetition, definition, dense)) or \ + len(repetition) != rows or len(definition) != rows or \ + any(value != 0 for value in repetition) or \ + any(not isinstance(value, int) or isinstance(value, bool) or + not 0 <= value <= maximum_definition + for value in definition) or \ + len(dense) != sum(value == maximum_definition + for value in definition) or \ + not isinstance(group.get("compression"), str) or \ + not isinstance(group.get("pages"), list): + raise HarnessError(f"Arrow Rust level evidence differs: {fixture['id']}") + current_rows.append(rows) + if row_group_rows is None: + row_group_rows = current_rows + elif row_group_rows != current_rows: + raise HarnessError(f"Arrow Rust leaf row counts differ: {fixture['id']}") + if evidence.get("rows") != sum(row_group_rows or []): + raise HarnessError(f"Arrow Rust total row count differs: {fixture['id']}") + return None + + +def validated_arrow_rows(evidence, leaves, case_id): + arrow = evidence["arrow"] + names = [leaf["path"][0] for leaf in leaves] + if any(len(leaf["path"]) != 1 for leaf in leaves) or \ + len(set(names)) != len(names): + raise HarnessError(f"Arrow Rust logical fixture is not flat: {case_id}") + rows = [] + for canonical_row, encoded_row in zip(arrow["canonical_rows"], + arrow["json_rows"]): + if not isinstance(canonical_row, list) or \ + len(canonical_row) != len(leaves) or \ + not isinstance(encoded_row, str): + raise HarnessError(f"Arrow Rust canonical row differs: {case_id}") + row = load_arrow_json_row(encoded_row) + if list(row) != names: + raise HarnessError(f"Arrow Rust JSON field order differs: {case_id}") + for index, entry in enumerate(canonical_row): + if not isinstance(entry, dict) or set(entry) != {"field", "value"} or \ + entry["field"] != names[index]: + raise HarnessError(f"Arrow Rust canonical field differs: {case_id}") + rows.append((canonical_row, row)) + return names, rows + + +def validate_audit(document, fixture, raw_columns, descriptor, audit_name): + expected_root = { + "evidence_version", "oracle", "action", "file_count", + "supported_count", "unsupported_count", "files", + } + if set(document) != expected_root or document["evidence_version"] != 1 or \ + document["action"] != "audit" or document["file_count"] != 1 or \ + document["supported_count"] != 1 or \ + document["unsupported_count"] != 0: + raise HarnessError(f"Arrow Rust audit envelope differs: {fixture['id']}") + validate_tool_evidence(document["oracle"], descriptor) + if not isinstance(document["files"], list) or len(document["files"]) != 1: + raise HarnessError(f"Arrow Rust audit file coverage differs: {fixture['id']}") + result = document["files"][0] + if set(result) != {"status", "file", "evidence", "error"} or \ + result["status"] != "supported" or result["error"] is not None: + raise HarnessError(f"Arrow Rust audit rejected fixture: {fixture['id']}") + if result["file"] != audit_name or not isinstance(result["evidence"], dict): + raise HarnessError(f"Arrow Rust audit filename differs: {fixture['id']}") + evidence = result["evidence"] + required = { + "case_id", "file_name", "sha256", "file_bytes", "rows", + "row_groups", "physical_schema", "columns", "arrow", + } + if set(evidence) != required or evidence["case_id"] != audit_name or \ + evidence["file_name"] != audit_name or \ + evidence["sha256"] != fixture["sha256"] or \ + evidence["file_bytes"] != fixture["size"] or \ + evidence["row_groups"] != fixture["row_group_count"] or \ + not isinstance(evidence["physical_schema"], str): + raise HarnessError(f"Arrow Rust file facts differ: {fixture['id']}") + leaves = leaf_records(fixture["id"], raw_columns) + validate_audit_columns(evidence, leaves, fixture, raw_columns) + arrow = evidence["arrow"] + expected_arrow = { + "status", "schema", "canonical_rows", "json_rows", + "ordered_map_rows", "diagnostic", + } + if not isinstance(arrow, dict) or set(arrow) != expected_arrow or \ + arrow["status"] != "ok" or arrow["ordered_map_rows"] is not None or \ + arrow["diagnostic"] is not None: + raise HarnessError(f"Arrow Rust high-level read differs: {fixture['id']}") + rows = evidence["rows"] + if not isinstance(rows, int) or isinstance(rows, bool) or rows < 0 or \ + not isinstance(arrow["canonical_rows"], list) or \ + not isinstance(arrow["json_rows"], list) or \ + len(arrow["canonical_rows"]) != rows or \ + len(arrow["json_rows"]) != rows: + raise HarnessError(f"Arrow Rust row evidence differs: {fixture['id']}") + schema = arrow["schema"] + if not isinstance(schema, list) or len(schema) != len(leaves): + raise HarnessError(f"Arrow Rust schema coverage differs: {fixture['id']}") + for leaf, field, column in zip(leaves, schema, evidence["columns"]): + if set(field) != {"name", "nullable", "data_type", "metadata"} or \ + field["name"] != leaf["path"][0] or \ + field["data_type"] != expected_arrow_type(leaf) or \ + field["nullable"] != \ + (column["maximum_definition_level"] > 0) or \ + not isinstance(field["metadata"], dict) or \ + any(not isinstance(key, str) or not isinstance(value, str) + for key, value in field["metadata"].items()): + raise HarnessError(f"Arrow Rust Arrow schema differs: {fixture['id']}") + validated_arrow_rows(evidence, leaves, fixture["id"]) + return evidence, leaves + + +def validate_metadata(document, fixture, descriptor, audit_name): + expected = { + "oracle", "action", "file_name", "file_bytes", "row_group_count", + "leaf_count", "row_groups", + } + if not isinstance(document, dict) or set(document) != expected or \ + document["action"] != "type-order" or \ + document["file_name"] != audit_name or \ + document["file_bytes"] != fixture["size"] or \ + document["row_group_count"] != fixture["row_group_count"] or \ + document["leaf_count"] != fixture["leaf_count"]: + raise HarnessError( + f"Arrow Rust metadata file facts differ: {fixture['id']}") + validate_tool_evidence(document["oracle"], descriptor) + groups = document["row_groups"] + if not isinstance(groups, list) or \ + len(groups) != fixture["row_group_count"]: + raise HarnessError( + f"Arrow Rust metadata row-group coverage differs: {fixture['id']}") + for row_group, group in enumerate(groups): + if not isinstance(group, dict) or set(group) != { + "row_group", "row_count", "columns"} or \ + group["row_group"] != row_group or \ + not isinstance(group["row_count"], int) or \ + isinstance(group["row_count"], bool) or \ + group["row_count"] < 0 or \ + not isinstance(group["columns"], list) or \ + len(group["columns"]) != fixture["leaf_count"]: + raise HarnessError( + f"Arrow Rust metadata row group differs: {fixture['id']}") + for column in group["columns"]: + if not isinstance(column, dict) or set(column) != { + "path", "physical_type", "column_order", "num_values"} or \ + not isinstance(column["path"], list) or \ + not column["path"] or \ + any(not isinstance(part, str) or not part + for part in column["path"]) or \ + not isinstance(column["physical_type"], str) or \ + column["column_order"] not in ( + "TYPE_ORDER", "UNDEFINED", "UNKNOWN") or \ + not isinstance(column["num_values"], int) or \ + isinstance(column["num_values"], bool) or \ + column["num_values"] < 0: + raise HarnessError( + f"Arrow Rust metadata column differs: {fixture['id']}") + return document + + +def type_order_observations(document, case_id, raw_file, raw_columns): + columns = [record for (current, _, _), record in raw_columns.items() + if current == case_id] + columns.sort(key=lambda record: (record["row_group"], record["leaf"])) + groups = document["row_groups"] + if len(columns) != sum(len(group["columns"]) for group in groups): + raise HarnessError("Arrow Rust TYPE_ORDER topology differs") + for raw in columns: + observed = groups[raw["row_group"]]["columns"][raw["leaf"]] + if observed["column_order"] != "TYPE_ORDER" or \ + raw["column_order"]["state"] != "TYPE_ORDER": + raise HarnessError("Arrow Rust column order is not TYPE_ORDER") + if observed["path"] != raw["path"] or \ + observed["physical_type"] != \ + raw["leaf_schema"]["physical_type"] or \ + observed["num_values"] != int(raw["num_values"]): + raise HarnessError("Arrow Rust TYPE_ORDER facts differ") + return [{"file": raw_file, "columns": columns}] + + +def timestamp_nanoseconds(value): + match = TIMESTAMP_PATTERN.fullmatch(value) + if match is None: + raise HarnessError(f"unsupported Arrow Rust timestamp: {value!r}") + year, month, day, hour, minute, second = map(int, match.groups()[:6]) + base = datetime.datetime(year, month, day, hour, minute, second) + epoch = datetime.datetime(1970, 1, 1) + delta = base - epoch + fraction = (match.group(7) or "").ljust(9, "0") + return str((delta.days * 86400 + delta.seconds) * 1_000_000_000 + + int(fraction or "0")) + + +def canonical_leaf_value(value, json_value, leaf, arrow_type): + if value is None: + if json_value is not None: + raise HarnessError("Arrow Rust null representations disagree") + return None + schema = leaf["leaf_schema"] + physical = schema["physical_type"] + logical = schema["logical_type"] + if logical == "DECIMAL": + if not isinstance(value, dict) or \ + value.get("data_type") != arrow_type or \ + set(value) != {"data_type", "display"} or \ + not isinstance(value["display"], str): + raise HarnessError("Arrow Rust DECIMAL representation differs") + parsed = decimal.Decimal(value["display"]) + if decimal.Decimal(json_value) != parsed: + raise HarnessError("Arrow Rust DECIMAL JSON representation differs") + return {"decimal_scale": schema["scale"], + "unscaled": decimal_unscaled(parsed, schema["scale"])} + if physical in ("FLOAT", "DOUBLE"): + digits = 8 if physical == "FLOAT" else 16 + if not isinstance(value, str) or not re.fullmatch( + rf"0x[0-9a-f]{{{digits}}}", value): + raise HarnessError("Arrow Rust floating representation differs") + if not isinstance(json_value, (int, decimal.Decimal)) or \ + isinstance(json_value, bool): + raise HarnessError("Arrow Rust floating JSON representation differs") + return {"float32_bits" if physical == "FLOAT" else "float64_bits": + value[2:]} + if physical == "INT96": + if not isinstance(value, dict) or set(value) != \ + {"data_type", "display"} or value["data_type"] != arrow_type or \ + not isinstance(value["display"], str) or \ + json_value != value["display"]: + raise HarnessError("Arrow Rust INT96 representation differs") + return {"timestamp_nanoseconds": timestamp_nanoseconds(value["display"])} + if physical in ("BYTE_ARRAY", "FIXED_LEN_BYTE_ARRAY") and \ + logical not in ("JSON", "STRING", "ENUM"): + if not isinstance(json_value, str) or len(json_value) % 2 or \ + HEX_PATTERN.fullmatch(json_value) is None: + raise HarnessError("Arrow Rust binary representation differs") + if physical == "FIXED_LEN_BYTE_ARRAY" and \ + len(json_value) != 2 * schema["type_length"]: + raise HarnessError("Arrow Rust fixed binary width differs") + return {"bytes_hex": json_value} + if logical in ("JSON", "STRING", "ENUM"): + if not isinstance(value, str) or json_value != value: + raise HarnessError("Arrow Rust string representation differs") + return value + if physical == "BOOLEAN": + if not isinstance(value, bool) or json_value is not value: + raise HarnessError("Arrow Rust BOOLEAN representation differs") + return value + if physical in ("INT32", "INT64"): + if not isinstance(value, int) or isinstance(value, bool) or \ + json_value != value: + raise HarnessError("Arrow Rust integer representation differs") + return value + raise HarnessError( + f"unsupported Arrow Rust logical value: {physical}/{logical}") + + +def logical_observations(evidence, leaves, case_id): + arrow = evidence["arrow"] + names, rows = validated_arrow_rows(evidence, leaves, case_id) + columns = [[] for _ in leaves] + arrow_types = [field["data_type"] for field in arrow["schema"]] + for canonical_row, row in rows: + for index, (entry, leaf, arrow_type) in enumerate(zip(canonical_row, + leaves, arrow_types)): + columns[index].append(canonical_leaf_value(entry["value"], + row[names[index]], leaf, arrow_type)) + normalized = [] + for leaf, values in zip(leaves, columns): + normalized.append({ + "logical_type": leaf["leaf_schema"]["logical_type"], + "path": leaf["path"], + "physical_type": leaf["leaf_schema"]["physical_type"], + "values": values, + }) + return [{ + "columns": normalized, + "contract": "n6-logical-values-v1", + "row_count": evidence["rows"], + }] + + +def validate_claim_scope(claims): + expected = { + (case_id, "read.logical-values"): "planned" + for case_id in EXPECTED_LOGICAL_CASES + } + expected.update({key: "unsupported" for key in EXPECTED_UNSUPPORTED}) + type_order = {(case_id, "wire.column-order.type") + for case_id in EXPECTED_TYPE_ORDER_CASES} + claimed_type_order = {key for key in claims + if key[1] == "wire.column-order.type"} + if claimed_type_order and claimed_type_order != type_order: + raise HarnessError("Arrow Rust TYPE_ORDER capability scope differs") + if claimed_type_order: + expected.update({key: "planned" for key in type_order}) + if set(claims) != set(expected): + raise HarnessError("Arrow Rust capability claim scope differs") + for key, status in claims.items(): + if expected[key] == "planned" and status not in ("planned", "verified"): + raise HarnessError(f"Arrow Rust positive claim status differs: {key}") + if expected[key] == "unsupported" and status != "unsupported": + raise HarnessError(f"Arrow Rust unsupported claim status differs: {key}") + return None + + +def generate(arguments): + requested_repository = checked_directory(arguments.repository, + "repository root") + for path, label in ((arguments.manifest, "manifest"), + (arguments.capabilities, "capabilities"), + (arguments.fixtures, "fixtures"), + (arguments.descriptor, "toolchain descriptor"), + (arguments.raw_evidence, "normalized raw evidence")): + checked_regular_file(path, label) + context = build_context(arguments, PRODUCER, EVIDENCE_ID, + draft_evidence=arguments.draft) + repository = context["repository"] + if repository != requested_repository: + raise HarnessError("repository root changed during context construction") + manifest = context["manifest"] + authority_record = context["authority_record"] + known = context["known"] + claims = context["claims"] + raw_files = context["raw_files"] + raw_columns = context["raw_columns"] + limits = context["limits"] + descriptor_sha256 = context["descriptor_sha256"] + descriptor = context["descriptor"] + expected_descriptor = repository_input(repository, DESCRIPTOR_RELATIVE) + if pathlib.Path(arguments.descriptor).resolve(strict=True) != \ + expected_descriptor: + raise HarnessError("descriptor path does not select the Arrow Rust pin") + validate_descriptor(repository, descriptor, authority_record) + validate_claim_scope(claims) + corpus_root = checked_directory(arguments.corpus_root, "fixture root") + selected_cases = sorted({case_id for case_id, _ in claims}) + expected_cases = EXPECTED_LOGICAL_CASES | \ + {case_id for case_id, _ in EXPECTED_UNSUPPORTED} + type_order_enabled = any(capability == "wire.column-order.type" + for _, capability in claims) + if type_order_enabled: + expected_cases |= EXPECTED_TYPE_ORDER_CASES + if set(selected_cases) != expected_cases: + raise HarnessError("Arrow Rust fixture coverage differs") + total_bytes = sum(known[case_id]["size"] for case_id in selected_cases) + if total_bytes > limits["max_total_bytes"]: + raise HarnessError("Arrow Rust fixture bytes exceed the evidence limit") + output = checked_output_path(arguments.output) + try: + output.relative_to(repository) + except ValueError: + pass + else: + declared_output = repository.joinpath( + *pathlib.PurePosixPath(context["target_entry"]["file"]).parts) + if output != declared_output: + raise HarnessError( + "repository output is not the declared evidence path") + metadata_binary = repository_input(repository, + descriptor["metadata_binary_file"]) + reject_output_alias(output, + [*context["protected_inputs"], metadata_binary], [corpus_root]) + docker_image_identity(arguments, descriptor) + verify_image_files(arguments, descriptor) + unsupported_cases = {case_id for (case_id, _), status in claims.items() + if status == "unsupported"} + records = [run_record(EVIDENCE_ID, PRODUCER, authority_record, + descriptor_sha256, repository, manifest, unsupported_cases, + context["upstream_evidence"], input_hashes=context["input_hashes"])] + snapshots = tempfile.TemporaryDirectory(prefix="parquet-n6-arrow-rs-", + dir=output.parent) + snapshot_root = pathlib.Path(snapshots.name) + try: + snapshot_paths = {} + for case_id in selected_cases: + fixture = known[case_id] + if case_id not in raw_files: + raise HarnessError(f"raw file fact is absent: {case_id}") + path = checked_file(corpus_root, fixture["file"], fixture["sha256"], + fixture["size"], snapshot_root) + if path.stat().st_size > limits["max_file_bytes"]: + raise HarnessError( + f"Arrow Rust fixture exceeds its limit: {case_id}") + snapshot_paths[case_id] = path + metadata_snapshot = snapshot_metadata_binary(repository, descriptor, + snapshot_root) + for path in snapshot_paths.values(): + os.chmod(path, 0o444) + os.chmod(metadata_snapshot, 0o555) + os.chmod(snapshot_root, 0o555) + for case_id in selected_cases: + fixture = known[case_id] + path = snapshot_paths[case_id] + current_claims = sorted((capability, status) + for (current, capability), status in claims.items() + if current == case_id) + needs_audit = any(capability == "read.logical-values" or + status == "unsupported" for capability, status in current_claims) + evidence = None + leaves = None + if needs_audit: + command = audit_command(arguments, descriptor, snapshot_root, + path.name) + stdout, _ = run_bounded(command) + document = load_oracle_json(stdout) + evidence, leaves = validate_audit(document, fixture, + raw_columns, descriptor, path.name) + metadata = None + if any(capability == "wire.column-order.type" + for capability, _ in current_claims): + command = metadata_command(arguments, descriptor, + snapshot_root, path.name, metadata_snapshot.name) + stdout, _ = run_bounded(command) + metadata = validate_metadata(load_oracle_json(stdout), fixture, + descriptor, path.name) + records.append(raw_files[case_id]) + for capability, status in current_claims: + if status == "unsupported": + records.append(case_result(case_id, capability, + fixture["digest_contract"], "UNSUPPORTED")) + elif capability == "read.logical-values": + observations = logical_observations(evidence, leaves, + case_id) + records.append(case_result(case_id, capability, + fixture["digest_contract"], "PASS", observations)) + elif capability == "wire.column-order.type": + observations = type_order_observations(metadata, case_id, + raw_files[case_id], raw_columns) + records.append(case_result(case_id, capability, + fixture["digest_contract"], "PASS", observations)) + else: + raise HarnessError( + f"unhandled Arrow Rust capability: {capability}") + finally: + try: + os.chmod(snapshot_root, 0o700) + except FileNotFoundError: + pass + snapshots.cleanup() + expected_records = 1 + len(selected_cases) + len(claims) + if len(records) != expected_records: + raise HarnessError("Arrow Rust normalized record count differs") + value = evidence_bytes(records, limits["max_file_bytes"], + limits["max_line_bytes"], + limits["max_records_per_input"]) + atomic_output(output, value, arguments.check) + return value + + +def parser(): + result = argparse.ArgumentParser() + result.add_argument("--repository", required=True) + result.add_argument("--manifest", required=True) + result.add_argument("--capabilities", required=True) + result.add_argument("--fixtures", required=True) + result.add_argument("--descriptor", required=True) + result.add_argument("--raw-evidence", required=True) + result.add_argument("--corpus-root", required=True) + result.add_argument("--output", required=True) + result.add_argument("--docker", default="docker") + result.add_argument("--check", action="store_true") + result.add_argument("--draft", action="store_true") + return result + + +def main(): + arguments = parser().parse_args() + value = generate(arguments) + mode = "checked" if arguments.check else "wrote" + print(f"{EVIDENCE_ID}: {mode} {value.count(b'\n')} records and " + f"{len(value)} bytes") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as error: + print(f"N6 Arrow Rust harness failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/test/conformance/n6/oracles/arrow-rs/run.sh b/test/conformance/n6/oracles/arrow-rs/run.sh new file mode 100755 index 0000000..b5eeeb4 --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/run.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +if [ "$#" -lt 3 ]; then + echo "usage: run.sh CORPUS_ROOT RAW_EVIDENCE OUTPUT [--draft]" >&2 + exit 2 +fi + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +repository=$(CDPATH= cd -- "$oracle_dir/../../../../.." && pwd -P) +if [ -z "${PARQUET_N6_INTEROP_PYTHON_ROOT:-}" ]; then + echo "PARQUET_N6_INTEROP_PYTHON_ROOT is required" >&2 + exit 2 +fi +python="$PARQUET_N6_INTEROP_PYTHON_ROOT/bin/python3.12" +if [ ! -x "$python" ]; then + echo "the pinned Python interpreter is not executable: $python" >&2 + exit 2 +fi +corpus_root=$1 +raw_evidence=$2 +output=$3 +shift 3 + +exec "$python" -I -B -S "$oracle_dir/run.py" \ + --repository "$repository" \ + --manifest "$repository/test/conformance/n6/manifest.toml" \ + --capabilities "$repository/test/conformance/n6/capabilities.toml" \ + --fixtures "$repository/test/conformance/n6/fixtures.toml" \ + --descriptor "$oracle_dir/toolchain.toml" \ + --raw-evidence "$raw_evidence" \ + --corpus-root "$corpus_root" \ + --output "$output" \ + "$@" diff --git a/test/conformance/n6/oracles/arrow-rs/runtests.py b/test/conformance/n6/oracles/arrow-rs/runtests.py new file mode 100755 index 0000000..50fa515 --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/runtests.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +import argparse +import json +import pathlib +import sys +import tempfile +import tomllib +import types +import unittest + +SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIRECTORY)) +import run as harness +sys.path.pop(0) + + +EXPECTED_LOGICAL_DIGESTS = { + "apache-alltypes-dictionary": + "f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64", + "apache-alltypes-plain": + "7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3", + "apache-binary": + "abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603", + "apache-bson": + "eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a", + "apache-byte-array-decimal": + "72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345", + "apache-fixed-length-byte-array": + "b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43", + "apache-fixed-length-decimal": + "695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24", + "apache-fixed-length-decimal-legacy": + "ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb", + "apache-int32-decimal": + "457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788", + "apache-int32-with-null-pages": + "40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7", + "apache-int64-decimal": + "995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab", + "apache-json": + "34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9", + "apache-rle-boolean-encoding": + "898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa", +} +EXPECTED_TYPE_ORDER_DIGESTS = { + "apache-binary": + "398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2", + "apache-binary-truncated-min-max": + "386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae", + "apache-bson": + "5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c", + "apache-fixed-length-byte-array": + "a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af", + "apache-float16-nonzeros-and-nans": + "495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b", + "apache-float16-zeros-and-nans": + "1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665", + "apache-int32-with-null-pages": + "febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a", + "apache-json": + "f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04", + "apache-nan-in-stats": + "3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7", + "apache-single-nan": + "6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d", +} + + +def leaf(physical, logical="NONE", **fields): + schema = { + "physical_type": physical, + "logical_type": logical, + "converted_type": None, + "type_length": None, + "precision": None, + "scale": None, + "bit_width": None, + "is_signed": None, + "time_unit": None, + "is_adjusted_to_utc": None, + "crs": None, + "geography_algorithm": None, + } + schema.update(fields) + return {"leaf_schema": schema, "path": ["value"]} + + +class ArrowRustHarnessTests(unittest.TestCase): + def test_timestamp_normalization_is_exact_to_nanoseconds(self): + self.assertEqual(harness.timestamp_nanoseconds("1970-01-01T00:00:00"), + "0") + self.assertEqual(harness.timestamp_nanoseconds( + "1970-01-01T00:00:00.000000001"), "1") + self.assertEqual(harness.timestamp_nanoseconds( + "1969-12-31T23:59:59.999999999"), "-1") + with self.assertRaises(harness.HarnessError): + harness.timestamp_nanoseconds("1970-01-01T00:00:00Z") + + def test_canonical_values_preserve_bits_bytes_and_decimals(self): + decimal_leaf = leaf("INT32", "DECIMAL", converted_type="DECIMAL", + precision=4, scale=2) + value = {"data_type": "Decimal128(4, 2)", "display": "-1.25"} + self.assertEqual(harness.canonical_leaf_value(value, -1.25, + decimal_leaf, "Decimal128(4, 2)"), + {"decimal_scale": 2, "unscaled": "-125"}) + float_leaf = leaf("FLOAT") + self.assertEqual(harness.canonical_leaf_value("0x80000000", + harness.decimal.Decimal("-0.0"), float_leaf, "Float32"), + {"float32_bits": "80000000"}) + binary_leaf = leaf("BYTE_ARRAY") + self.assertEqual(harness.canonical_leaf_value("ignored", "00ff", + binary_leaf, "Binary"), {"bytes_hex": "00ff"}) + fixed_leaf = leaf("FIXED_LEN_BYTE_ARRAY", type_length=2) + self.assertEqual(harness.canonical_leaf_value( + {"data_type": "FixedSizeBinary(2)", "display": "00ff"}, "00ff", + fixed_leaf, "FixedSizeBinary(2)"), {"bytes_hex": "00ff"}) + with self.assertRaises(harness.HarnessError): + harness.canonical_leaf_value("ignored", "00", fixed_leaf, + "FixedSizeBinary(2)") + + def test_scalar_and_int96_values_are_checked(self): + self.assertIs(harness.canonical_leaf_value(True, True, + leaf("BOOLEAN"), "Boolean"), True) + self.assertEqual(harness.canonical_leaf_value(7, 7, leaf("INT32"), + "Int32"), 7) + timestamp = { + "data_type": "Timestamp(Nanosecond, None)", + "display": "1970-01-01T00:00:00.000000001", + } + self.assertEqual(harness.canonical_leaf_value(timestamp, + timestamp["display"], leaf("INT96"), timestamp["data_type"]), + {"timestamp_nanoseconds": "1"}) + self.assertIsNone(harness.canonical_leaf_value(None, None, + leaf("INT32"), "Int32")) + + def test_docker_audit_has_no_network_and_read_only_inputs(self): + arguments = types.SimpleNamespace(docker="docker") + descriptor = { + "image_platform": "linux/amd64", + "binary_path": "/usr/local/bin/oracle", + "image_id": "sha256:" + "1" * 64, + "image_reference": "image:tag", + } + command = harness.audit_command(arguments, descriptor, + pathlib.Path("/private/tmp/corpus"), "data/input.parquet") + self.assertIn("none", command) + self.assertEqual(command[command.index("--network") + 1], "none") + self.assertEqual(command[command.index("--pull") + 1], "never") + self.assertIn("--read-only", command) + self.assertIn("no-new-privileges", command) + self.assertIn(descriptor["image_id"], command) + self.assertNotIn(descriptor["image_reference"], command) + mount = command[command.index("--mount") + 1] + self.assertTrue(mount.endswith(",target=/corpus,readonly")) + metadata = harness.metadata_command(arguments, descriptor, + pathlib.Path("/private/tmp/corpus"), "data/input.parquet", + ".metadata-oracle") + self.assertIn(descriptor["image_id"], metadata) + self.assertNotIn(descriptor["image_reference"], metadata) + self.assertEqual(metadata[metadata.index("--entrypoint") + 1], + "/corpus/.metadata-oracle") + with self.assertRaises(harness.HarnessError): + harness.audit_command(arguments, descriptor, + pathlib.Path("/private/tmp/bad,corpus"), "data/input.parquet") + + def test_type_order_claim_scope_is_exact(self): + claims = {(case_id, "read.logical-values"): "planned" + for case_id in harness.EXPECTED_LOGICAL_CASES} + claims.update({key: "unsupported" + for key in harness.EXPECTED_UNSUPPORTED}) + self.assertIsNone(harness.validate_claim_scope(claims)) + claims.update({(case_id, "wire.column-order.type"): "planned" + for case_id in harness.EXPECTED_TYPE_ORDER_CASES}) + self.assertIsNone(harness.validate_claim_scope(claims)) + del claims[(next(iter(harness.EXPECTED_TYPE_ORDER_CASES)), + "wire.column-order.type")] + with self.assertRaises(harness.HarnessError): + harness.validate_claim_scope(claims) + + def test_type_order_observations_are_normalized(self): + raw_file = {"case_id": "case", "record": "file"} + raw_column = { + "case_id": "case", + "column_order": {"state": "TYPE_ORDER"}, + "leaf": 0, + "num_values": "3", + "path": ["value"], + "leaf_schema": {"physical_type": "BYTE_ARRAY"}, + "record": "column_statistics", + "row_group": 0, + } + document = { + "row_groups": [{ + "columns": [{ + "column_order": "TYPE_ORDER", + "has_min_max": True, + "max_hex": "ff", + "min_hex": "00", + "null_count": 1, + "num_values": 3, + "path": ["value"], + "physical_type": "BYTE_ARRAY", + }], + "row_group": 0, + "row_count": 3, + }], + } + raw_columns = {("case", 0, 0): raw_column} + expected = [{"file": raw_file, "columns": [raw_column]}] + self.assertEqual(harness.type_order_observations(document, "case", + raw_file, raw_columns), expected) + document["row_groups"][0]["columns"][0]["column_order"] = \ + "UNDEFINED" + with self.assertRaises(harness.HarnessError): + harness.type_order_observations(document, "case", raw_file, + raw_columns) + + def test_bounded_process_caps_stdout_and_rejects_failure(self): + stdout, stderr = harness.run_bounded( + [sys.executable, "-c", "print('ok')"], 16, 16, 5) + self.assertEqual(stdout, b"ok\n") + self.assertEqual(stderr, b"") + with self.assertRaises(harness.HarnessError): + harness.run_bounded([sys.executable, "-c", + "import os; os.write(1, b'x' * 1024)"], 16, 16, 5) + with self.assertRaises(harness.HarnessError): + harness.run_bounded([sys.executable, "-c", + "raise SystemExit(3)"], 16, 1024, 5) + + def test_output_rejects_symbolic_link_components(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory).resolve(strict=True) + target = root / "target" + target.mkdir() + linked = root / "linked" + linked.symlink_to(target, target_is_directory=True) + with self.assertRaises(harness.HarnessError): + harness.checked_output_path(linked / "evidence.jsonl") + output = target / "evidence.jsonl" + self.assertEqual(harness.checked_output_path(output), output) + destination_link = target / "linked.jsonl" + destination_link.symlink_to(output) + with self.assertRaises(harness.HarnessError): + harness.checked_output_path(destination_link) + + def test_input_rejects_symbolic_link_components(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory).resolve(strict=True) + target = root / "target" + target.mkdir() + source = target / "input.parquet" + source.write_bytes(b"PAR1") + self.assertEqual(harness.checked_regular_file(source, "input"), + source) + linked_file = target / "linked.parquet" + linked_file.symlink_to(source) + with self.assertRaises(harness.HarnessError): + harness.checked_regular_file(linked_file, "input") + linked_directory = root / "linked" + linked_directory.symlink_to(target, target_is_directory=True) + with self.assertRaises(harness.HarnessError): + harness.checked_regular_file(linked_directory / source.name, + "input") + with self.assertRaises(harness.HarnessError): + harness.checked_directory(linked_directory, "input root") + + def test_toolchain_descriptor_pins_current_wrapper_sources(self): + repository = SCRIPT_DIRECTORY.parents[4] + authority_record = { + "version": "59.2.0", + "revision": "782e5a685501a9db6cc8e9a3b7cbff894940c47a", + } + with (SCRIPT_DIRECTORY / "toolchain.toml").open("rb") as stream: + descriptor_input = tomllib.load(stream) + descriptor = harness.validate_descriptor(repository, + descriptor_input, authority_record) + self.assertEqual(descriptor["producer"], "arrow-rs") + self.assertEqual(len(descriptor["source"]), 12) + self.assertEqual(len(descriptor["wrapper"]), 8) + + +def integration_arguments(arguments, output, check): + repository = pathlib.Path(arguments.repository).resolve(strict=True) + return types.SimpleNamespace( + repository=str(repository), + manifest=str(repository / "test/conformance/n6/manifest.toml"), + capabilities=str(repository / "test/conformance/n6/capabilities.toml"), + fixtures=str(repository / "test/conformance/n6/fixtures.toml"), + descriptor=str(SCRIPT_DIRECTORY / "toolchain.toml"), + raw_evidence=arguments.raw_evidence, + corpus_root=arguments.corpus_root, + output=str(output), + docker=arguments.docker, + check=check, + draft=arguments.draft, + ) + + +def validate_integration_records(path): + records = [json.loads(line) for line in path.read_text().splitlines()] + if len(records) != 45: + raise AssertionError(f"expected 45 records, got {len(records)}") + logical = { + record["case_id"]: record["actual_sha256"] + for record in records + if record["record"] == "case_result" and + record["capability_id"] == "read.logical-values" + } + if logical != EXPECTED_LOGICAL_DIGESTS: + raise AssertionError("Arrow Rust logical digests differ") + type_order = { + record["case_id"]: record["actual_sha256"] + for record in records + if record["record"] == "case_result" and + record["capability_id"] == "wire.column-order.type" + } + if type_order != EXPECTED_TYPE_ORDER_DIGESTS: + raise AssertionError("Arrow Rust TYPE_ORDER digests differ") + unsupported = { + (record["case_id"], record["capability_id"]) + for record in records + if record["record"] == "case_result" and + record["status"] == "UNSUPPORTED" + } + if unsupported != harness.EXPECTED_UNSUPPORTED: + raise AssertionError("Arrow Rust unsupported results differ") + if len({record["case_id"] for record in records + if record["record"] == "file"}) != 19: + raise AssertionError("Arrow Rust file coverage differs") + return None + + +def run_integration(arguments): + repository = pathlib.Path(arguments.repository).resolve(strict=True) + output = repository / \ + "test/conformance/n6/evidence/arrow-rs.normalized.jsonl" + harness.generate(integration_arguments(arguments, output, False)) + first = output.read_bytes() + validate_integration_records(output) + harness.generate(integration_arguments(arguments, output, True)) + if output.read_bytes() != first: + raise AssertionError("Arrow Rust check mode changed the output") + print(f"Arrow Rust integration passed: 45 records, {len(first)} bytes") + return None + + +def parser(): + result = argparse.ArgumentParser() + result.add_argument("--integration", action="store_true") + result.add_argument("--repository") + result.add_argument("--corpus-root") + result.add_argument("--raw-evidence") + result.add_argument("--docker", default="docker") + result.add_argument("--draft", action="store_true") + return result + + +def main(): + arguments = parser().parse_args() + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + ArrowRustHarnessTests) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if not result.wasSuccessful(): + return 1 + if arguments.integration: + required = (arguments.repository, arguments.corpus_root, + arguments.raw_evidence) + if any(value is None for value in required): + raise SystemExit( + "--integration requires --repository, --corpus-root, and --raw-evidence") + run_integration(arguments) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/conformance/n6/oracles/arrow-rs/toolchain.toml b/test/conformance/n6/oracles/arrow-rs/toolchain.toml new file mode 100644 index 0000000..3897941 --- /dev/null +++ b/test/conformance/n6/oracles/arrow-rs/toolchain.toml @@ -0,0 +1,103 @@ +descriptor_version = 1 +status = "verified" +producer = "arrow-rs" +producer_version = "59.2.0" +source_revision = "782e5a685501a9db6cc8e9a3b7cbff894940c47a" +image_reference = "parquet-jl-n5-oracles:n5d-canonical-a" +image_id = "sha256:04e56f6512080165bff7c9ae869b27ab20b14fd2dbb7fd7dc3827bc2a1a6568b" +image_platform = "linux/amd64" +binary_path = "/usr/local/bin/parquet-jl-n5-arrow-rs-oracle" +binary_sha256 = "be68c3de1ef338e64966cfc632ab71be68a6646c16d205d5e2a7fcbd8d51885c" +metadata_binary_file = "test/conformance/n6/oracles/arrow-rs/build/parquet-jl-n6-arrow-rs-metadata" +metadata_binary_sha256 = "a26a0a29f99adde346800e85ee66544226d2105a040b40eb3536e8d629d8c301" +metadata_binary_size = 710064 +platform = "macos-15-arm64" +python_version = "3.12.8" +python_distribution_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250115/cpython-3.12.8%2B20250115-aarch64-apple-darwin-install_only_stripped.tar.gz" +python_distribution_sha256 = "dfb8a4c87116538717105ef3dec3668ae07590a5b5532109fec3ccad90be2fbc" +python_tree_policy = "extract-strip-site-packages-bytecode-v1" +python_executable_sha256 = "d6b64f766d3b08326aa10cdb37c9d922e3af38b57ec07caa28b894e5fccf6e69" +python_tree_sha256 = "e3b7dcdffba67f605b0fa3318656387e8d4265d34531c2bdbc43d3aba4a033ec" +rust_toolchain = "1.96.1" +rustc = "rustc 1.96.1 (31fca3adb 2026-06-26)" +cargo = "cargo 1.96.1 (356927216 2026-06-26)" + +[[source]] +path = "/opt/bootstrap/arrow-rs/.gitignore" +sha256 = "88f688c06fa1b7a099d148c3448775bdfaee09a65efe2d88072d2a8c38a9df97" + +[[source]] +path = "/opt/bootstrap/arrow-rs/Cargo.lock" +sha256 = "5797b71e20f0a4eda601c8c126336e4daffa2dfc1d9a992d6762cd3fc0b902cc" + +[[source]] +path = "/opt/bootstrap/arrow-rs/Cargo.toml" +sha256 = "f33fdb977c0281b98554bf1e701aebe8bd012d8166e9b354a013ac81ed54696f" + +[[source]] +path = "/opt/bootstrap/arrow-rs/README.md" +sha256 = "6e3c26f9ef32fe2f0503761437e40a2acecdc0b2c4c8490a787c70e510263a23" + +[[source]] +path = "/opt/bootstrap/arrow-rs/UPSTREAM.toml" +sha256 = "d547dbed70c01b77f076a6f020767693751d869c3ff5f5279c5d6ce06f152218" + +[[source]] +path = "/opt/bootstrap/arrow-rs/rust-toolchain.toml" +sha256 = "2a64c46431c4bd1e02f1d4bbcbe3390e74dd4668faf9d3cb8da8663efd371add" + +[[source]] +path = "/opt/bootstrap/arrow-rs/scripts/build.sh" +sha256 = "7aaaecaf6b37b2b139aa2ff2d3c7bc8e1d5c089749fc620da2f0079ae3dd8311" + +[[source]] +path = "/opt/bootstrap/arrow-rs/scripts/run.sh" +sha256 = "2cdedbc41fb639af8a739bc253e36e0fad2f668dcfbadf620cda0f40bdd8c00f" + +[[source]] +path = "/opt/bootstrap/arrow-rs/scripts/test.sh" +sha256 = "55c150ce78f3f1a9b9f00758a725a0bb369f5021435df5c0f36667078333db3e" + +[[source]] +path = "/opt/bootstrap/arrow-rs/src/cases.rs" +sha256 = "b5e5a12e7e21fe5bf6de85ae43a2a9062b36beaa683419d0df46d9a744cd49bf" + +[[source]] +path = "/opt/bootstrap/arrow-rs/src/evidence.rs" +sha256 = "b37cc66e4fa258b5e47ba4103ce190e24b7fb8c5cbdce0d1083da7ecab469730" + +[[source]] +path = "/opt/bootstrap/arrow-rs/src/main.rs" +sha256 = "172ff23c1ba569fc1d5170fc183152a376acc450a119c5ef330f9edcf7edde12" + +[[wrapper]] +path = "test/conformance/n6/harnesses/common.py" +sha256 = "8b48613a795deed37089f6dbfca8eb6ed18062ebfd907a4606c91ef3308776ad" + +[[wrapper]] +path = "test/conformance/n6/oracles/arrow-rs/build.sh" +sha256 = "7d2a79ea00d64ac50eea6e866a6ff94b20275e2bcdb27d7256b16070def316c7" + +[[wrapper]] +path = "test/conformance/n6/oracles/arrow-rs/run.py" +sha256 = "992c487acf716a324cc49713c5ac1aefbc5a5b195bc66047830e9a16e0da833c" + +[[wrapper]] +path = "test/conformance/n6/oracles/arrow-rs/run.sh" +sha256 = "b386e4b83c599ee646beb72df02efcd5c61e77ef9b69e8fa1c4a0bc3fb1df084" + +[[wrapper]] +path = "test/conformance/n6/oracles/arrow-rs/runtests.py" +sha256 = "a3857a242c9bad0f3a3405fd6222389a7f66b0726a547e7a24a470975e040c43" + +[[wrapper]] +path = "test/conformance/n6/oracles/arrow-rs/check.sh" +sha256 = "453fc2b3a93d45c90f43bcb4684ed0726999529fb44dbc3f567db8fc416fa56d" + +[[wrapper]] +path = "test/conformance/n6/oracles/arrow-rs/metadata/Cargo.toml" +sha256 = "a7b9439da961d319d828c758badebb9f93d1d85625f2461e7867c34848d060cc" + +[[wrapper]] +path = "test/conformance/n6/oracles/arrow-rs/metadata/src/main.rs" +sha256 = "a2b50a6b20153dee9a4a8d48d8adae5f223b430909b90c6b208eaae3e067e9a8" diff --git a/test/conformance/n6/oracles/parquet-java/.gitignore b/test/conformance/n6/oracles/parquet-java/.gitignore new file mode 100644 index 0000000..82f4bb9 --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/.gitignore @@ -0,0 +1,2 @@ +/build/ +/__pycache__/ diff --git a/test/conformance/n6/oracles/parquet-java/README.md b/test/conformance/n6/oracles/parquet-java/README.md new file mode 100644 index 0000000..ae82439 --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/README.md @@ -0,0 +1,91 @@ +# N6 Parquet Java interoperability harness + +This harness checks the reviewed Parquet Java 1.17.1 N6 capability scope. It +uses the release runtime whose jar manifest names exact source commit +`78a8d3230eb4769db93de5f2f2e18363c04cae81`. It also binds the source files +that implement local reading, legacy statistics conversion, version parsing, +and PARQUET-251 policy. + +The run path does not use Maven, a network, an ambient Java executable, an +ambient classpath, or an ambient Python installation. It uses the pinned +Temurin 21.0.8+9 tree already selected for the N6 raw Java scanner. It uses +three exact Maven Central jar inputs copied into the ignored `build/` cache by +`build.sh`. The Java audit process receives a minimal environment, explicit +heap and metaspace limits, read-only snapshots, bounded output, and a timeout. + +The base logical and compatibility scope has 34 records. It covers 14 files, 13 passing +`read.logical-values` results, four fixture-backed +safe legacy-bound suppression results, one standalone +`producer-parquet-251` policy result, and one explicit `UNSUPPORTED` result for +`wire.statistics.nan-count`. The full reviewed scope adds ten passing +`wire.column-order.type` results and five new file records. The total is then +49 records. Each positive result compares Parquet Java's exposed column order, +path, physical type, and value count with the frozen raw facts before it emits +the shared raw observation envelope. The policy result has no fake file record. + +## Prepare the offline cache + +Obtain these exact upstream artifacts outside the repository. `build.sh` does +not download them and rejects every wrong byte identity. + +- `parquet-cli-1.17.1-runtime.jar`, SHA-256 + `d0173051493c506a298c691e555a41a682a405895fd0c8cc429a7e1cb1fcc711` +- `hadoop-client-api-3.3.0.jar`, SHA-256 + `d549ba6d131fd6c8e5d42a78dab5c790950edd6258523dedc556b537ca6654aa` +- `hadoop-client-runtime-3.3.0.jar`, SHA-256 + `2ba23f1e1dbb03e73600a41fcb187ad2626529684ed226085a91b0a0d6d67ee5` + +Then build the deterministic Java harness jar. The build rejects symbolic links +and non-directory cache paths before it writes any output. + +```sh +export PARQUET_N6_JAVA_JDK_ROOT=/path/to/pinned/temurin-21.0.8+9 +export PARQUET_N6_PARQUET_CLI_JAR=/path/to/parquet-cli-1.17.1-runtime.jar +export PARQUET_N6_HADOOP_CLIENT_API_JAR=/path/to/hadoop-client-api-3.3.0.jar +export PARQUET_N6_HADOOP_CLIENT_RUNTIME_JAR=/path/to/hadoop-client-runtime-3.3.0.jar +test/conformance/n6/oracles/parquet-java/build.sh +``` + +## Run a draft + +Draft mode accepts a planned evidence entry. It does not relax any input, +source, binary, or descriptor digest check. The descriptor must have the exact +hash authorized in `capabilities.toml`. The normalized raw input must select the +exact repository input declared by `manifest.toml`. + +```sh +export PARQUET_N6_INTEROP_PYTHON_ROOT=/path/to/reviewed/cpython-3.12.8 +export PARQUET_N6_JAVA_ROOT=/path/to/parquet-java-1.17.1 +export PARQUET_N6_JAVA_JDK_ROOT=/path/to/pinned/temurin-21.0.8+9 +test/conformance/n6/oracles/parquet-java/run.sh \ + /path/to/parquet-testing \ + test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl \ + /private/tmp/parquet-java.normalized.jsonl \ + --draft +test/conformance/n6/oracles/parquet-java/check.sh \ + /path/to/parquet-testing \ + test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl \ + /private/tmp/parquet-java.normalized.jsonl \ + --draft +``` + +Run unit tests and the two-pass integration rehearsal: + +```sh +"$PARQUET_N6_INTEROP_PYTHON_ROOT/bin/python3.12" -I -S -B \ + test/conformance/n6/oracles/parquet-java/runtests.py \ + --integration \ + --repository . \ + --corpus-root /path/to/parquet-testing \ + --raw-evidence test/conformance/n6/evidence/raw-java-apache-corpus.normalized.jsonl \ + --java-root "$PARQUET_N6_JAVA_ROOT" \ + --jdk-root "$PARQUET_N6_JAVA_JDK_ROOT" \ + --draft +``` + +The integration command writes the exact manifest target at +`test/conformance/n6/evidence/parquet-java.normalized.jsonl`, then checks it +without changing its bytes. + +This directory does not authorize evidence freeze, oracle publication, +`oracles.lock`, or a release claim. diff --git a/test/conformance/n6/oracles/parquet-java/build.sh b/test/conformance/n6/oracles/parquet-java/build.sh new file mode 100755 index 0000000..146e7f4 --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/build.sh @@ -0,0 +1,123 @@ +#!/bin/sh +set -eu + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +build_root="$oracle_dir/build" +artifact_dir="$build_root/artifacts" + +reject_directory_path() { + path=$1 + label=$2 + if [ -L "$path" ] || { [ -e "$path" ] && [ ! -d "$path" ]; }; then + echo "$label must be a real directory: $path" >&2 + exit 1 + fi +} + +reject_output_path() { + path=$1 + if [ -L "$path" ] || { [ -e "$path" ] && [ ! -f "$path" ]; }; then + echo "artifact output must be a regular non-link file: $path" >&2 + exit 1 + fi +} + +reject_directory_path "$build_root" "build root" +reject_directory_path "$artifact_dir" "artifact directory" +for output in \ + "$artifact_dir/parquet-cli-1.17.1-runtime.jar" \ + "$artifact_dir/hadoop-client-api-3.3.0.jar" \ + "$artifact_dir/hadoop-client-runtime-3.3.0.jar" \ + "$artifact_dir/parquet-java-n6-harness.jar"; do + reject_output_path "$output" +done +if [ -z "${PARQUET_N6_JAVA_JDK_ROOT:-}" ]; then + echo "PARQUET_N6_JAVA_JDK_ROOT is required" >&2 + exit 2 +fi +if [ -z "${PARQUET_N6_PARQUET_CLI_JAR:-}" ]; then + echo "PARQUET_N6_PARQUET_CLI_JAR is required" >&2 + exit 2 +fi +if [ -z "${PARQUET_N6_HADOOP_CLIENT_API_JAR:-}" ]; then + echo "PARQUET_N6_HADOOP_CLIENT_API_JAR is required" >&2 + exit 2 +fi +if [ -z "${PARQUET_N6_HADOOP_CLIENT_RUNTIME_JAR:-}" ]; then + echo "PARQUET_N6_HADOOP_CLIENT_RUNTIME_JAR is required" >&2 + exit 2 +fi +jdk_root=$(CDPATH= cd -- "$PARQUET_N6_JAVA_JDK_ROOT" && pwd -P) +java_source="$oracle_dir/src/org/julialang/parquet/n6/java/AuditMain.java" +parquet_jar=$PARQUET_N6_PARQUET_CLI_JAR +hadoop_api_jar=$PARQUET_N6_HADOOP_CLIENT_API_JAR +hadoop_runtime_jar=$PARQUET_N6_HADOOP_CLIENT_RUNTIME_JAR +expected_javac=7be7937fc6bae0ca89f0866f9ce94fc40a935dfb87806d3c701eca3402cfb90a +expected_jar=b4b69691321fb426e95a21bae51724e9f98ab6b67370eba953ee40c4f7512cbf +expected_parquet=d0173051493c506a298c691e555a41a682a405895fd0c8cc429a7e1cb1fcc711 +expected_parquet_size=50072122 +expected_hadoop_api=d549ba6d131fd6c8e5d42a78dab5c790950edd6258523dedc556b537ca6654aa +expected_hadoop_api_size=19207034 +expected_hadoop_runtime=2ba23f1e1dbb03e73600a41fcb187ad2626529684ed226085a91b0a0d6d67ee5 +expected_hadoop_runtime_size=27255121 +expected_harness=7d6de1067e4e01de65f5868c8643f4a68fe4ff6afc50759716d685bd7b9e764c + +sha256() { + shasum -a 256 "$1" | awk '{print $1}' +} + +verify() { + expected=$1 + file=$2 + actual=$(sha256 "$file") + if [ "$actual" != "$expected" ]; then + echo "SHA-256 mismatch: $file" >&2 + exit 1 + fi +} + +for input in "$parquet_jar" "$hadoop_api_jar" "$hadoop_runtime_jar"; do + [ ! -L "$input" ] && [ -f "$input" ] || { + echo "every Java artifact must be a regular non-link file" >&2 + exit 1 + } +done +[ "$(stat -f '%z' "$parquet_jar")" = "$expected_parquet_size" ] || exit 1 +[ "$(stat -f '%z' "$hadoop_api_jar")" = "$expected_hadoop_api_size" ] || exit 1 +[ "$(stat -f '%z' "$hadoop_runtime_jar")" = "$expected_hadoop_runtime_size" ] || exit 1 +verify "$expected_parquet" "$parquet_jar" +verify "$expected_hadoop_api" "$hadoop_api_jar" +verify "$expected_hadoop_runtime" "$hadoop_runtime_jar" +verify "$expected_javac" "$jdk_root/bin/javac" +verify "$expected_jar" "$jdk_root/bin/jar" + +mkdir -p "$build_root" +reject_directory_path "$build_root" "build root" +mkdir -p "$artifact_dir" +reject_directory_path "$artifact_dir" "artifact directory" +temporary=$(mktemp -d "$build_root/.build.XXXXXX") +trap 'rm -rf "$temporary"' EXIT HUP INT TERM +mkdir -p "$temporary/classes" +classpath="$parquet_jar:$hadoop_api_jar:$hadoop_runtime_jar" +"$jdk_root/bin/javac" --release 11 -encoding UTF-8 \ + -cp "$classpath" -d "$temporary/classes" "$java_source" +"$jdk_root/bin/jar" --create \ + --date=2020-01-01T00:00:00Z \ + --file "$temporary/parquet-java-n6-harness.jar" \ + -C "$temporary/classes" . +verify "$expected_harness" "$temporary/parquet-java-n6-harness.jar" +cp "$parquet_jar" "$temporary/parquet-cli-1.17.1-runtime.jar" +cp "$hadoop_api_jar" "$temporary/hadoop-client-api-3.3.0.jar" +cp "$hadoop_runtime_jar" "$temporary/hadoop-client-runtime-3.3.0.jar" +verify "$expected_parquet" "$temporary/parquet-cli-1.17.1-runtime.jar" +verify "$expected_hadoop_api" "$temporary/hadoop-client-api-3.3.0.jar" +verify "$expected_hadoop_runtime" "$temporary/hadoop-client-runtime-3.3.0.jar" +mv -f "$temporary/parquet-cli-1.17.1-runtime.jar" \ + "$artifact_dir/parquet-cli-1.17.1-runtime.jar" +mv -f "$temporary/hadoop-client-api-3.3.0.jar" \ + "$artifact_dir/hadoop-client-api-3.3.0.jar" +mv -f "$temporary/hadoop-client-runtime-3.3.0.jar" \ + "$artifact_dir/hadoop-client-runtime-3.3.0.jar" +mv -f "$temporary/parquet-java-n6-harness.jar" \ + "$artifact_dir/parquet-java-n6-harness.jar" +printf 'Built the pinned Parquet Java N6 harness.\n' diff --git a/test/conformance/n6/oracles/parquet-java/check.sh b/test/conformance/n6/oracles/parquet-java/check.sh new file mode 100755 index 0000000..b902d35 --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/check.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu + +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +exec "$oracle_dir/run.sh" "$@" --check + diff --git a/test/conformance/n6/oracles/parquet-java/run.py b/test/conformance/n6/oracles/parquet-java/run.py new file mode 100755 index 0000000..ba2128d --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/run.py @@ -0,0 +1,984 @@ +#!/usr/bin/env python3 +import argparse +import decimal +import io +import json +import os +import pathlib +import re +import selectors +import stat +import subprocess +import sys +import tempfile +import time +import zipfile + +SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parent +HARNESS_DIRECTORY = SCRIPT_DIRECTORY.parents[1] / "harnesses" +sys.path.insert(0, str(HARNESS_DIRECTORY)) +import common +sys.path = [entry for entry in sys.path + if pathlib.Path(entry or ".").resolve() != HARNESS_DIRECTORY] + + +HarnessError = common.HarnessError +PRODUCER = "parquet-java" +EVIDENCE_ID = "normalized-parquet-java" +POLICY_CASE = "producer-parquet-251" +DESCRIPTOR_RELATIVE = \ + "test/conformance/n6/oracles/parquet-java/toolchain.toml" +JAVA_MAIN = "org.julialang.parquet.n6.java.AuditMain" +JAVA_STDOUT_LIMIT = 8 * 1024 * 1024 +JAVA_STDERR_LIMIT = 1024 * 1024 +JAVA_TIMEOUT_SECONDS = 30 +EXPECTED_RECORD_COUNT = 49 +EXPECTED_LOGICAL_CASES = { + "apache-alltypes-dictionary", + "apache-alltypes-plain", + "apache-binary", + "apache-bson", + "apache-byte-array-decimal", + "apache-fixed-length-byte-array", + "apache-fixed-length-decimal", + "apache-fixed-length-decimal-legacy", + "apache-int32-decimal", + "apache-int32-with-null-pages", + "apache-int64-decimal", + "apache-json", + "apache-rle-boolean-encoding", +} +EXPECTED_TYPE_ORDER_CASES = { + "apache-binary", + "apache-binary-truncated-min-max", + "apache-bson", + "apache-fixed-length-byte-array", + "apache-float16-nonzeros-and-nans", + "apache-float16-zeros-and-nans", + "apache-int32-with-null-pages", + "apache-json", + "apache-nan-in-stats", + "apache-single-nan", +} +EXPECTED_LEGACY_CASES = { + "apache-fixed-length-decimal", + "apache-fixed-length-decimal-legacy", + "apache-int32-decimal", + "apache-int64-decimal", + POLICY_CASE, +} +EXPECTED_UNSUPPORTED = { + ("apache-floating-orders-nan-count", "wire.statistics.nan-count"), +} +EXPECTED_POLICY_FACTS = ( + ("null-binary", None, "BINARY", "not_applicable", "error", None, + None, None, True), + ("empty-binary", "", "BINARY", "not_applicable", "error", None, + None, None, True), + ("empty-application-binary", " version 1.0.0", "BINARY", + "not_applicable", "error", None, None, None, True), + ("unparsable-binary", "garbage!", "BINARY", "not_applicable", + "error", None, None, None, True), + ("unrelated-binary", "impala version 1.0.0", "BINARY", + "not_applicable", "parsed", "impala", "1.0.0", None, False), + ("missing-semver-binary", "parquet-mr version", "BINARY", + "not_applicable", "parsed", "parquet-mr", None, None, True), + ("before-fix-distinct-binary", "parquet-mr version 1.7.9", "BINARY", + "distinct", "parsed", "parquet-mr", "1.7.9", None, True), + ("before-fix-equal-binary", "parquet-mr version 1.7.9", "BINARY", + "equal", "parsed", "parquet-mr", "1.7.9", None, True), + ("release-candidate-binary", "parquet-mr version 1.8.0-rc1", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.8.0-rc1", None, + True), + ("fixed-binary", "parquet-mr version 1.8.0", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.8.0", None, False), + ("cdh-before-binary", "parquet-mr version 1.5.0-cdh5.4.9", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.5.0-cdh5.4.9", + None, True), + ("cdh-start-binary", "parquet-mr version 1.5.0-cdh5.5.0", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.5.0-cdh5.5.0", + None, False), + ("cdh-end-binary", "parquet-mr version 1.5.0", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.5.0", None, True), + ("before-fix-fixed", "parquet-mr version 1.7.9", + "FIXED_LEN_BYTE_ARRAY", "not_applicable", "parsed", "parquet-mr", + "1.7.9", None, True), + ("fixed-fixed", "parquet-mr version 1.8.0", + "FIXED_LEN_BYTE_ARRAY", "not_applicable", "parsed", "parquet-mr", + "1.8.0", None, False), + ("before-fix-int32", "parquet-mr version 1.7.9", "INT32", + "not_applicable", "parsed", "parquet-mr", "1.7.9", None, False), +) +EXPECTED_SOURCE_FILES = { + "parquet-common/src/main/java/org/apache/parquet/VersionParser.java", + "parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java", + "parquet-common/src/main/java/org/apache/parquet/io/LocalInputFile.java", + "parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java", + "parquet-column/src/main/java/org/apache/parquet/example/data/Group.java", + "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java", + "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java", + "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/example/GroupReadSupport.java", + "parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java", +} +EXPECTED_WRAPPERS = { + "test/conformance/n6/harnesses/common.py", + "test/conformance/n6/oracles/parquet-java/build.sh", + "test/conformance/n6/oracles/parquet-java/check.sh", + "test/conformance/n6/oracles/parquet-java/run.py", + "test/conformance/n6/oracles/parquet-java/run.sh", + "test/conformance/n6/oracles/parquet-java/runtests.py", + "test/conformance/n6/oracles/parquet-java/src/org/julialang/parquet/n6/java/AuditMain.java", +} +EXPECTED_ARTIFACTS = { + "hadoop-client-api", + "hadoop-client-runtime", + "parquet-cli-runtime", +} +HEX_PATTERN = re.compile(r"^[0-9a-f]*$") + + +def expected_policy_observations(): + observations = [] + for scenario, created_by, physical, relation, status, application, \ + version, build, ignored in EXPECTED_POLICY_FACTS: + version_parse = {"status": status} + if status == "parsed": + version_parse.update({ + "application": application, + "version": version, + "build": build, + }) + observations.append({ + "scenario_id": scenario, + "created_by": created_by, + "physical_type": physical, + "bound_relation": relation, + "version_parse": version_parse, + "should_ignore_statistics": ignored, + }) + return observations + + +def reject_symlink_components(path, label, allow_missing_leaf=False): + absolute = pathlib.Path(os.path.abspath(path)) + current = pathlib.Path(absolute.anchor) + parts = absolute.parts[1:] if absolute.anchor else absolute.parts + for index, part in enumerate(parts): + current = current / part + try: + metadata = current.lstat() + except FileNotFoundError: + if allow_missing_leaf and index == len(parts) - 1: + return absolute + raise HarnessError(f"{label} does not exist: {current}") + if stat.S_ISLNK(metadata.st_mode): + raise HarnessError(f"{label} contains a symbolic link: {current}") + return absolute + + +def checked_directory(path, label): + path = reject_symlink_components(path, label) + if not path.is_dir(): + raise HarnessError(f"{label} is not a directory") + return path.resolve(strict=True) + + +def checked_regular_file(path, label): + path = reject_symlink_components(path, label) + if not path.is_file(): + raise HarnessError(f"{label} is not a regular file") + return path.resolve(strict=True) + + +def checked_output_path(path): + requested = pathlib.Path(os.path.abspath(path)) + checked_directory(requested.parent, "evidence output parent") + return reject_symlink_components(requested, "evidence output", + allow_missing_leaf=True) + + +def canonical_integer(value): + if value != "0" and (value.startswith("0") or value.startswith("-0")): + raise HarnessError(f"noncanonical JSON integer: {value}") + return int(value) + + +def unique_object(pairs): + output = {} + for key, value in pairs: + if key in output: + raise HarnessError(f"duplicate JSON object key: {key}") + output[key] = value + return output + + +def invalid_number(value): + raise HarnessError(f"invalid JSON number: {value}") + + +def load_oracle_json(value): + try: + decoded = value.decode("utf-8") + result = json.loads(decoded, object_pairs_hook=unique_object, + parse_constant=invalid_number, parse_float=invalid_number, + parse_int=canonical_integer) + except (UnicodeError, json.JSONDecodeError) as error: + raise HarnessError("Parquet Java returned invalid JSON") from error + if not isinstance(result, dict): + raise HarnessError("Parquet Java did not return a JSON object") + return result + + +def run_bounded(command, environment, working_directory, + maximum_stdout=JAVA_STDOUT_LIMIT, + maximum_stderr=JAVA_STDERR_LIMIT, + timeout_seconds=JAVA_TIMEOUT_SECONDS): + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment, + cwd=working_directory, close_fds=True) + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ, "stdout") + selector.register(process.stderr, selectors.EVENT_READ, "stderr") + output = {"stdout": bytearray(), "stderr": bytearray()} + limits = {"stdout": maximum_stdout, "stderr": maximum_stderr} + deadline = time.monotonic() + timeout_seconds + try: + while selector.get_map(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise HarnessError("bounded Java command timed out") + events = selector.select(remaining) + if not events: + raise HarnessError("bounded Java command timed out") + for key, _ in events: + chunk = os.read(key.fileobj.fileno(), 65536) + if not chunk: + selector.unregister(key.fileobj) + continue + name = key.data + output[name].extend(chunk) + if len(output[name]) > limits[name]: + raise HarnessError( + f"bounded Java command {name} exceeds its limit") + status = process.wait(timeout=max(0.1, deadline - time.monotonic())) + except BaseException: + process.kill() + process.wait() + raise + finally: + selector.close() + if status != 0: + diagnostic = bytes(output["stderr"]).decode( + "utf-8", "replace").strip() + if len(diagnostic) > 1000: + diagnostic = diagnostic[:1000] + "..." + raise HarnessError( + f"bounded Java command failed with status {status}: {diagnostic}") + return bytes(output["stdout"]), bytes(output["stderr"]) + + +def validate_claim_scope(claims): + expected = { + (case_id, "read.logical-values"): "planned" + for case_id in EXPECTED_LOGICAL_CASES + } + expected.update({ + (case_id, "compat.legacy-statistics"): "planned" + for case_id in EXPECTED_LEGACY_CASES + }) + expected.update({key: "unsupported" for key in EXPECTED_UNSUPPORTED}) + type_order = {(case_id, "wire.column-order.type") + for case_id in EXPECTED_TYPE_ORDER_CASES} + claimed_type_order = {key for key in claims + if key[1] == "wire.column-order.type"} + if claimed_type_order and claimed_type_order != type_order: + raise HarnessError( + "Parquet Java TYPE_ORDER capability scope differs") + if claimed_type_order: + expected.update({key: "planned" for key in type_order}) + if set(claims) != set(expected): + raise HarnessError("Parquet Java capability claim scope differs") + for key, expected_status in expected.items(): + status = claims[key] + if expected_status == "planned" and status not in ( + "planned", "verified"): + raise HarnessError(f"Parquet Java claim status differs: {key}") + if expected_status == "unsupported" and status != "unsupported": + raise HarnessError(f"Parquet Java unsupported status differs: {key}") + return None + + +def validate_identity_list(items, expected_paths, key): + if not isinstance(items, list) or not items: + raise HarnessError(f"Parquet Java {key} identity list is empty") + observed = set() + for item in items: + if set(item) != {key, "sha256"} or item[key] in observed or \ + re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) is None: + raise HarnessError(f"Parquet Java {key} identity is invalid") + observed.add(item[key]) + if observed != expected_paths: + raise HarnessError(f"Parquet Java {key} coverage differs") + return None + + +def validate_descriptor(repository, descriptor, authority_record): + required = { + "descriptor_version", "status", "producer", "producer_version", + "source_revision", "platform", "python_version", + "python_distribution_url", "python_distribution_sha256", + "python_tree_policy", "python_executable_sha256", + "python_tree_sha256", "java_vendor", + "java_version", "java_executable_sha256", "javac_executable_sha256", + "java_release_sha256", "jdk_tree_sha256", "hadoop_version", + "harness_main", "harness_jar_file", "harness_jar_sha256", + "harness_jar_size", "artifact", "source", "wrapper", + } + if set(descriptor) != required or descriptor["descriptor_version"] != 1: + raise HarnessError("Parquet Java descriptor has invalid keys") + expected = { + "producer": PRODUCER, + "producer_version": authority_record["version"], + "source_revision": authority_record["revision"], + "platform": "macos-15-arm64", + "python_version": "3.12.8", + "java_vendor": "Eclipse-Adoptium-Temurin", + "java_version": "21.0.8+9", + "hadoop_version": "3.3.0", + "harness_main": JAVA_MAIN, + } + for field, value in expected.items(): + if descriptor[field] != value: + raise HarnessError(f"Parquet Java descriptor has stale {field}") + if descriptor["status"] not in ("planned", "verified"): + raise HarnessError("Parquet Java descriptor has invalid status") + for field in ("python_executable_sha256", "python_tree_sha256", + "java_executable_sha256", "javac_executable_sha256", + "java_release_sha256", "jdk_tree_sha256", + "harness_jar_sha256"): + if re.fullmatch(r"[0-9a-f]{64}", descriptor[field]) is None: + raise HarnessError(f"Parquet Java descriptor has invalid {field}") + if not isinstance(descriptor["harness_jar_size"], int) or \ + isinstance(descriptor["harness_jar_size"], bool) or \ + descriptor["harness_jar_size"] <= 0: + raise HarnessError("Parquet Java harness jar size is invalid") + validate_identity_list(descriptor["source"], EXPECTED_SOURCE_FILES, "file") + validate_identity_list(descriptor["wrapper"], EXPECTED_WRAPPERS, "path") + for item in descriptor["wrapper"]: + candidate = common.repository_input(repository, item["path"]) + if common.sha256_file(candidate) != item["sha256"]: + raise HarnessError( + f"Parquet Java wrapper digest differs: {item['path']}") + artifacts = descriptor["artifact"] + if not isinstance(artifacts, list) or \ + {item.get("id") for item in artifacts} != EXPECTED_ARTIFACTS: + raise HarnessError("Parquet Java artifact coverage differs") + for item in artifacts: + if set(item) != {"id", "file", "url", "sha256", "size"} or \ + re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) is None or \ + not isinstance(item["size"], int) or \ + isinstance(item["size"], bool) or item["size"] <= 0 or \ + not item["url"].startswith( + "https://repo.maven.apache.org/maven2/"): + raise HarnessError("Parquet Java artifact identity is invalid") + common.repository_input(repository, item["file"]) + common.repository_input(repository, descriptor["harness_jar_file"]) + common.verify_python_runtime(descriptor) + return descriptor + + +def validate_java_toolchain(descriptor, java_root, jdk_root): + if common.sha256_file(jdk_root / "bin/java") != \ + descriptor["java_executable_sha256"]: + raise HarnessError("Parquet Java runtime executable differs") + if common.sha256_file(jdk_root / "bin/javac") != \ + descriptor["javac_executable_sha256"]: + raise HarnessError("Parquet Java compiler executable differs") + if common.sha256_file(jdk_root / "release") != \ + descriptor["java_release_sha256"]: + raise HarnessError("Parquet Java JDK release file differs") + if common.tree_sha256(jdk_root) != descriptor["jdk_tree_sha256"]: + raise HarnessError("Parquet Java JDK tree differs") + for item in descriptor["source"]: + candidate = java_root.joinpath( + *pathlib.PurePosixPath(item["file"]).parts) + checked_regular_file(candidate, "Parquet Java authority source") + if common.sha256_file(candidate) != item["sha256"]: + raise HarnessError( + f"Parquet Java authority source differs: {item['file']}") + return None + + +def snapshot_input(path, expected_sha256, expected_size, root, name): + value = common.regular_file_bytes(path, expected_size, + f"toolchain artifact {name}") + if len(value) != expected_size or \ + common.sha256_bytes(value) != expected_sha256: + raise HarnessError(f"Parquet Java artifact identity differs: {name}") + destination = pathlib.Path(root) / name + descriptor = os.open(destination, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + try: + os.unlink(destination) + except FileNotFoundError: + pass + raise + return destination, value + + +def verify_parquet_manifest(value, descriptor): + try: + with zipfile.ZipFile(io.BytesIO(value)) as archive: + names = archive.namelist() + if len(names) != len(set(names)): + raise HarnessError("Parquet CLI jar has duplicate entries") + manifest = archive.read("META-INF/MANIFEST.MF").decode("utf-8") + except (KeyError, UnicodeError, zipfile.BadZipFile) as error: + raise HarnessError("Parquet CLI jar manifest is invalid") from error + expected = { + f"Implementation-Version: {descriptor['producer_version']}", + f"git-SHA-1: {descriptor['source_revision']}", + } + lines = set(manifest.replace("\r\n", "\n").splitlines()) + if not expected <= lines: + raise HarnessError("Parquet CLI jar authority differs") + return None + + +def snapshot_toolchain(repository, descriptor, root): + output = {} + for item in descriptor["artifact"]: + source = common.repository_input(repository, item["file"]) + path, value = snapshot_input(source, item["sha256"], item["size"], + root, pathlib.PurePosixPath(item["file"]).name) + output[item["id"]] = path + if item["id"] == "parquet-cli-runtime": + verify_parquet_manifest(value, descriptor) + harness_source = common.repository_input(repository, + descriptor["harness_jar_file"]) + harness, _ = snapshot_input(harness_source, + descriptor["harness_jar_sha256"], descriptor["harness_jar_size"], + root, pathlib.PurePosixPath(descriptor["harness_jar_file"]).name) + output["harness"] = harness + for path in output.values(): + os.chmod(path, 0o444) + return output + + +def java_command(jdk_root, toolchain, action, path=None): + classpath = os.pathsep.join(str(toolchain[name]) for name in ( + "harness", "parquet-cli-runtime", "hadoop-client-api", + "hadoop-client-runtime")) + if any(character in classpath for character in ("\n", "\r")): + raise HarnessError("Parquet Java classpath is invalid") + command = [ + str(jdk_root / "bin/java"), + "-Xms16m", + "-Xmx256m", + "-XX:MaxMetaspaceSize=128m", + "-XX:+ExitOnOutOfMemoryError", + "-Dfile.encoding=UTF-8", + "-Duser.language=en", + "-Duser.country=US", + "-Duser.timezone=UTC", + "-cp", + classpath, + JAVA_MAIN, + action, + ] + if path is not None: + command.append(str(path)) + return command + + +def java_environment(jdk_root): + return { + "JAVA_HOME": str(jdk_root), + "LANG": "C", + "LC_ALL": "C", + "TZ": "UTC", + } + + +def validate_oracle_identity(document, descriptor, action): + expected = { + "name": PRODUCER, + "version": descriptor["producer_version"], + "commit": descriptor["source_revision"], + } + if document.get("oracle") != expected or document.get("action") != action: + raise HarnessError("Parquet Java oracle identity differs") + return None + + +def java_physical(physical): + return "BINARY" if physical == "BYTE_ARRAY" else physical + + +def validate_audit(document, fixture, leaves, raw_columns, descriptor): + expected_keys = { + "oracle", "action", "created_by", "schema", "row_count", + "columns", "row_groups", + } + if set(document) != expected_keys: + raise HarnessError(f"Parquet Java audit keys differ: {fixture['id']}") + validate_oracle_identity(document, descriptor, "audit") + schema = document["schema"] + if not isinstance(schema, list) or len(schema) != len(leaves): + raise HarnessError(f"Parquet Java schema coverage differs: {fixture['id']}") + for leaf, field in zip(leaves, schema): + required = {"name", "physical_type", "maximum_definition_level", + "maximum_repetition_level", "column_order"} + if set(field) != required or len(leaf["path"]) != 1 or \ + field["name"] != leaf["path"][0] or \ + field["physical_type"] != \ + java_physical(leaf["leaf_schema"]["physical_type"]) or \ + not isinstance(field["maximum_definition_level"], int) or \ + isinstance(field["maximum_definition_level"], bool) or \ + field["maximum_definition_level"] < 0 or \ + field["maximum_repetition_level"] != 0 or \ + field["column_order"] not in ( + "TYPE_DEFINED_ORDER", "UNDEFINED"): + raise HarnessError(f"Parquet Java schema differs: {fixture['id']}") + expected_rows = sum(int(raw_columns[(fixture["id"], group, 0)][ + "num_values"]) for group in range(fixture["row_group_count"])) + if document["row_count"] != expected_rows or \ + not isinstance(document["columns"], list) or \ + len(document["columns"]) != len(leaves) or \ + any(not isinstance(column, list) or len(column) != expected_rows + for column in document["columns"]): + raise HarnessError(f"Parquet Java row coverage differs: {fixture['id']}") + validate_audit_statistics(document, fixture, leaves, raw_columns) + return None + + +def validate_audit_statistics(document, fixture, leaves, raw_columns): + groups = document["row_groups"] + if not isinstance(groups, list) or \ + len(groups) != fixture["row_group_count"]: + raise HarnessError( + f"Parquet Java row-group coverage differs: {fixture['id']}") + for group_index, group in enumerate(groups): + if set(group) != {"row_group", "row_count", "columns"} or \ + group["row_group"] != group_index or \ + not isinstance(group["columns"], list) or \ + len(group["columns"]) != len(leaves): + raise HarnessError( + f"Parquet Java row-group facts differ: {fixture['id']}") + expected_rows = int(raw_columns[(fixture["id"], group_index, 0)][ + "num_values"]) + if group["row_count"] != expected_rows: + raise HarnessError( + f"Parquet Java row-group rows differ: {fixture['id']}") + for leaf_index, (leaf, column) in enumerate(zip( + leaves, group["columns"])): + required = {"path", "physical_type", "statistics_class", "empty", + "has_non_null_value", "num_nulls_set", "num_nulls", + "min_hex", "max_hex", "should_ignore_statistics"} + raw = raw_columns[(fixture["id"], group_index, leaf_index)] + if set(column) != required or column["path"] != leaf["path"] or \ + column["physical_type"] != \ + java_physical(leaf["leaf_schema"]["physical_type"]) or \ + not isinstance(column["statistics_class"], str) or \ + not all(isinstance(column[field], bool) for field in ( + "empty", "has_non_null_value", "num_nulls_set", + "should_ignore_statistics")): + raise HarnessError( + f"Parquet Java statistics identity differs: {fixture['id']}") + for field in ("min_hex", "max_hex"): + value = column[field] + if value is not None and (not isinstance(value, str) or + len(value) % 2 or HEX_PATTERN.fullmatch(value) is None): + raise HarnessError( + f"Parquet Java statistics bytes differ: {fixture['id']}") + if column["num_nulls"] is not None and ( + not isinstance(column["num_nulls"], int) or + isinstance(column["num_nulls"], bool) or + column["num_nulls"] < 0): + raise HarnessError( + f"Parquet Java statistics count differs: {fixture['id']}") + if raw["null_count"] is not None and column["num_nulls_set"] and \ + column["num_nulls"] != int(raw["null_count"]): + raise HarnessError( + f"Parquet Java null count differs: {fixture['id']}") + return None + + +def parse_canonical_integer(value, label): + if not isinstance(value, str) or re.fullmatch(r"-?(?:0|[1-9][0-9]*)", + value) is None: + raise HarnessError(f"Parquet Java {label} is not canonical") + return int(value) + + +def canonical_value(value, leaf): + if value is None: + return None + if not isinstance(value, dict) or set(value) != {"kind", "value"}: + raise HarnessError("Parquet Java tagged value is invalid") + schema = leaf["leaf_schema"] + physical = schema["physical_type"] + logical = schema["logical_type"] + kind = value["kind"] + encoded = value["value"] + if physical == "BOOLEAN": + if kind != "boolean" or not isinstance(encoded, bool): + raise HarnessError("Parquet Java BOOLEAN representation differs") + return encoded + if physical in ("INT32", "INT64"): + expected = "int32" if physical == "INT32" else "int64" + if kind != expected: + raise HarnessError("Parquet Java integer representation differs") + integer = parse_canonical_integer(encoded, physical) + if logical == "DECIMAL": + return {"decimal_scale": schema["scale"], + "unscaled": str(integer)} + return integer + if physical in ("FLOAT", "DOUBLE"): + expected = "float32_bits" if physical == "FLOAT" else "float64_bits" + digits = 8 if physical == "FLOAT" else 16 + if kind != expected or not isinstance(encoded, str) or \ + re.fullmatch(rf"[0-9a-f]{{{digits}}}", encoded) is None: + raise HarnessError("Parquet Java floating representation differs") + return {expected: encoded} + if physical in ("BYTE_ARRAY", "FIXED_LEN_BYTE_ARRAY", "INT96"): + expected = {"BYTE_ARRAY": "binary", "FIXED_LEN_BYTE_ARRAY": "fixed", + "INT96": "int96"}[physical] + if kind != expected or not isinstance(encoded, str) or \ + len(encoded) % 2 or HEX_PATTERN.fullmatch(encoded) is None: + raise HarnessError("Parquet Java binary representation differs") + raw = bytes.fromhex(encoded) + if physical == "FIXED_LEN_BYTE_ARRAY" and \ + len(raw) != schema["type_length"]: + raise HarnessError("Parquet Java fixed-width value differs") + if physical == "INT96": + if len(raw) != 12: + raise HarnessError("Parquet Java INT96 width differs") + nanoseconds = int.from_bytes(raw[:8], "little", signed=False) + if nanoseconds >= 86_400_000_000_000: + raise HarnessError("Parquet Java INT96 time-of-day differs") + julian_day = int.from_bytes(raw[8:], "little", signed=False) + epoch = (julian_day - 2_440_588) * 86_400_000_000_000 + \ + nanoseconds + return {"timestamp_nanoseconds": str(epoch)} + if logical == "DECIMAL": + if not raw: + raise HarnessError("Parquet Java DECIMAL bytes are empty") + unscaled = int.from_bytes(raw, "big", signed=True) + return {"decimal_scale": schema["scale"], + "unscaled": str(unscaled)} + if logical in ("JSON", "STRING", "ENUM"): + try: + return raw.decode("utf-8", "strict") + except UnicodeError as error: + raise HarnessError( + "Parquet Java logical string is invalid UTF-8") from error + return {"bytes_hex": encoded} + raise HarnessError( + f"unsupported Parquet Java logical value: {physical}/{logical}") + + +def logical_observations(document, leaves): + columns = [] + for leaf, values in zip(leaves, document["columns"]): + columns.append({ + "logical_type": leaf["leaf_schema"]["logical_type"], + "path": leaf["path"], + "physical_type": leaf["leaf_schema"]["physical_type"], + "values": [canonical_value(value, leaf) for value in values], + }) + return [{ + "columns": columns, + "contract": "n6-logical-values-v1", + "row_count": document["row_count"], + }] + + +def type_order_observations(document, case_id, raw_file, raw_columns): + columns = [record for (current, _, _), record in raw_columns.items() + if current == case_id] + columns.sort(key=lambda record: (record["row_group"], record["leaf"])) + observed_groups = document["row_groups"] + if len(columns) != sum(len(group["columns"]) + for group in observed_groups): + raise HarnessError("Parquet Java TYPE_ORDER topology differs") + for raw in columns: + group = observed_groups[raw["row_group"]] + observed = group["columns"][raw["leaf"]] + schema = document["schema"][raw["leaf"]] + if schema["column_order"] != "TYPE_DEFINED_ORDER" or \ + raw["column_order"]["state"] != "TYPE_ORDER": + raise HarnessError( + "Parquet Java column order is not TYPE_ORDER") + if observed["path"] != raw["path"] or \ + observed["physical_type"] != \ + java_physical(raw["leaf_schema"]["physical_type"]) or \ + group["row_count"] != int(raw["num_values"]): + raise HarnessError("Parquet Java TYPE_ORDER facts differ") + return [{"file": raw_file, "columns": columns}] + + +def compatibility_observations(document, fixture, leaves, raw_columns): + columns = [] + expected_classes = { + "INT32": "org.apache.parquet.column.statistics.IntStatistics", + "INT64": "org.apache.parquet.column.statistics.LongStatistics", + "BYTE_ARRAY": "org.apache.parquet.column.statistics.BinaryStatistics", + "FIXED_LEN_BYTE_ARRAY": + "org.apache.parquet.column.statistics.BinaryStatistics", + } + for group in document["row_groups"]: + for leaf_index, (leaf, observed) in enumerate(zip( + leaves, group["columns"])): + raw = raw_columns[(fixture["id"], group["row_group"], leaf_index)] + physical = leaf["leaf_schema"]["physical_type"] + raw_min = raw["deprecated_min_hex"] + raw_max = raw["deprecated_max_hex"] + if observed["statistics_class"] != expected_classes[physical] or \ + observed["empty"] or \ + observed["has_non_null_value"] or \ + not observed["num_nulls_set"] or \ + observed["should_ignore_statistics"] or \ + observed["min_hex"] is not None or \ + observed["max_hex"] is not None or \ + raw_min is None or raw_max is None or \ + raw_min == raw_max or \ + observed["num_nulls"] != int(raw["null_count"]): + raise HarnessError( + f"Parquet Java legacy statistics differ: {fixture['id']}") + columns.append({ + "exposed_has_non_null_value": + observed["has_non_null_value"], + "exposed_max_hex": observed["max_hex"], + "exposed_min_hex": observed["min_hex"], + "num_nulls": observed["num_nulls"], + "path": leaf["path"], + "physical_type": physical, + "raw_deprecated_max_hex": raw_max, + "raw_deprecated_min_hex": raw_min, + "row_group": group["row_group"], + "producer_policy_ignored": + observed["should_ignore_statistics"], + "statistics_class": observed["statistics_class"], + }) + return [{ + "columns": columns, + "contract": "n6-parquet-java-legacy-statistics-v1", + "created_by": document["created_by"], + "row_group_count": len(document["row_groups"]), + }] + + +def validate_policy_document(document, descriptor): + if set(document) != {"oracle", "action", "observations"}: + raise HarnessError("Parquet Java policy keys differ") + validate_oracle_identity(document, descriptor, "policy") + observations = document["observations"] + expected = expected_policy_observations() + if not isinstance(observations, list) or len(observations) != len(expected): + raise HarnessError("Parquet Java policy coverage differs") + for observation, expected_observation in zip(observations, expected): + if observation != expected_observation: + raise HarnessError( + "Parquet Java policy observation differs: " + f"{expected_observation['scenario_id']}") + return observations + + +def run_java(jdk_root, toolchain, runtime_root, action, path=None): + command = java_command(jdk_root, toolchain, action, path) + stdout, _ = run_bounded(command, java_environment(jdk_root), runtime_root) + return load_oracle_json(stdout) + + +def generate(arguments): + requested_repository = checked_directory(arguments.repository, + "repository root") + for path, label in ((arguments.manifest, "manifest"), + (arguments.capabilities, "capabilities"), + (arguments.fixtures, "fixtures"), + (arguments.descriptor, "toolchain descriptor"), + (arguments.raw_evidence, "normalized raw evidence")): + checked_regular_file(path, label) + context = common.build_context(arguments, PRODUCER, EVIDENCE_ID, + draft_evidence=arguments.draft) + repository = context["repository"] + if repository != requested_repository: + raise HarnessError("repository root changed during context construction") + expected_descriptor = common.repository_input(repository, + DESCRIPTOR_RELATIVE) + if pathlib.Path(arguments.descriptor).resolve(strict=True) != \ + expected_descriptor: + raise HarnessError( + "descriptor path does not select the Parquet Java pin") + descriptor = validate_descriptor(repository, context["descriptor"], + context["authority_record"]) + validate_claim_scope(context["claims"]) + java_root = checked_directory(arguments.java_root, + "Parquet Java authority root") + jdk_root = checked_directory(arguments.jdk_root, "Parquet Java JDK root") + validate_java_toolchain(descriptor, java_root, jdk_root) + corpus_root = checked_directory(arguments.corpus_root, "fixture root") + selected_cases = sorted({case_id for case_id, _ in context["claims"] + if case_id != POLICY_CASE}) + expected_cases = EXPECTED_LOGICAL_CASES | \ + (EXPECTED_LEGACY_CASES - {POLICY_CASE}) | \ + {case_id for case_id, _ in EXPECTED_UNSUPPORTED} + if any(capability == "wire.column-order.type" + for _, capability in context["claims"]): + expected_cases |= EXPECTED_TYPE_ORDER_CASES + if set(selected_cases) != expected_cases: + raise HarnessError("Parquet Java fixture coverage differs") + total_bytes = sum(context["known"][case_id]["size"] + for case_id in selected_cases) + if total_bytes > context["limits"]["max_total_bytes"]: + raise HarnessError("Parquet Java fixtures exceed the evidence limit") + output = checked_output_path(arguments.output) + try: + output.relative_to(repository) + except ValueError: + pass + else: + declared_output = repository.joinpath( + *pathlib.PurePosixPath(context["target_entry"]["file"]).parts) + if output != declared_output: + raise HarnessError( + "repository output is not the declared evidence path") + artifact_inputs = [common.repository_input(repository, item["file"]) + for item in descriptor["artifact"]] + artifact_inputs.append(common.repository_input(repository, + descriptor["harness_jar_file"])) + common.reject_output_alias(output, + [*context["protected_inputs"], *artifact_inputs], + [corpus_root, java_root, jdk_root]) + unsupported_cases = {case_id for (case_id, _), status + in context["claims"].items() if status == "unsupported"} + records = [common.run_record(EVIDENCE_ID, PRODUCER, + context["authority_record"], context["descriptor_sha256"], + repository, context["manifest"], unsupported_cases, + context["upstream_evidence"], + input_hashes=context["input_hashes"])] + snapshots = tempfile.TemporaryDirectory(prefix="parquet-n6-java-inputs-", + dir=output.parent) + runtime = tempfile.TemporaryDirectory(prefix="parquet-n6-java-runtime-", + dir=output.parent) + snapshot_root = pathlib.Path(snapshots.name) + runtime_root = pathlib.Path(runtime.name) + try: + toolchain = snapshot_toolchain(repository, descriptor, snapshot_root) + snapshot_paths = {} + for case_id in selected_cases: + fixture = context["known"][case_id] + if case_id not in context["raw_files"]: + raise HarnessError(f"raw file fact is absent: {case_id}") + path = common.checked_file(corpus_root, fixture["file"], + fixture["sha256"], fixture["size"], snapshot_root) + if path.stat().st_size > context["limits"]["max_file_bytes"]: + raise HarnessError( + f"Parquet Java fixture exceeds its limit: {case_id}") + snapshot_paths[case_id] = path + for path in snapshot_paths.values(): + os.chmod(path, 0o444) + os.chmod(snapshot_root, 0o555) + for case_id in selected_cases: + fixture = context["known"][case_id] + records.append(context["raw_files"][case_id]) + current_claims = sorted((capability, status) + for (current, capability), status in context["claims"].items() + if current == case_id) + supported = any(status != "unsupported" + for _, status in current_claims) + document = None + leaves = common.leaf_records(case_id, context["raw_columns"]) + if supported: + document = run_java(jdk_root, toolchain, runtime_root, + "audit", snapshot_paths[case_id]) + validate_audit(document, fixture, leaves, + context["raw_columns"], descriptor) + if document["created_by"] != \ + context["raw_files"][case_id]["created_by"]: + raise HarnessError( + f"Parquet Java created_by differs: {case_id}") + for capability, status in current_claims: + if status == "unsupported": + records.append(common.case_result(case_id, capability, + fixture["digest_contract"], "UNSUPPORTED")) + elif capability == "read.logical-values": + records.append(common.case_result(case_id, capability, + fixture["digest_contract"], "PASS", + logical_observations(document, leaves))) + elif capability == "wire.column-order.type": + records.append(common.case_result(case_id, capability, + fixture["digest_contract"], "PASS", + type_order_observations(document, case_id, + context["raw_files"][case_id], + context["raw_columns"]))) + elif capability == "compat.legacy-statistics": + records.append(common.case_result(case_id, capability, + fixture["digest_contract"], "PASS", + compatibility_observations(document, fixture, leaves, + context["raw_columns"]))) + else: + raise HarnessError( + f"unhandled Parquet Java capability: {capability}") + policy_document = run_java(jdk_root, toolchain, runtime_root, "policy") + policy_observations = validate_policy_document(policy_document, + descriptor) + policy_case = context["known"][POLICY_CASE] + expected_policy = policy_case["expected_sha256"][ + "compat.legacy-statistics"] + records.append(common.case_result(POLICY_CASE, + "compat.legacy-statistics", policy_case["digest_contract"], + "PASS", policy_observations, expected_policy)) + finally: + try: + os.chmod(snapshot_root, 0o700) + except FileNotFoundError: + pass + snapshots.cleanup() + runtime.cleanup() + expected_records = 1 + len(selected_cases) + len(context["claims"]) + if len(records) != expected_records: + raise HarnessError("Parquet Java normalized record count differs") + value = common.evidence_bytes(records, + context["limits"]["max_file_bytes"], + context["limits"]["max_line_bytes"], + context["limits"]["max_records_per_input"]) + common.atomic_output(output, value, arguments.check) + return value + + +def parser(): + result = argparse.ArgumentParser() + result.add_argument("--repository", required=True) + result.add_argument("--manifest", required=True) + result.add_argument("--capabilities", required=True) + result.add_argument("--fixtures", required=True) + result.add_argument("--descriptor", required=True) + result.add_argument("--raw-evidence", required=True) + result.add_argument("--corpus-root", required=True) + result.add_argument("--java-root", required=True) + result.add_argument("--jdk-root", required=True) + result.add_argument("--output", required=True) + result.add_argument("--check", action="store_true") + result.add_argument("--draft", action="store_true") + return result + + +def main(): + try: + generate(parser().parse_args()) + except HarnessError as error: + print(f"parquet-java: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/conformance/n6/oracles/parquet-java/run.sh b/test/conformance/n6/oracles/parquet-java/run.sh new file mode 100755 index 0000000..26bdc86 --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/run.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +if [ "$#" -lt 3 ]; then + echo "usage: run.sh CORPUS_ROOT RAW_EVIDENCE OUTPUT [--draft]" >&2 + exit 2 +fi +oracle_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +repository=$(CDPATH= cd -- "$oracle_dir/../../../../.." && pwd -P) +if [ -z "${PARQUET_N6_INTEROP_PYTHON_ROOT:-}" ]; then + echo "PARQUET_N6_INTEROP_PYTHON_ROOT is required" >&2 + exit 2 +fi +if [ -z "${PARQUET_N6_JAVA_JDK_ROOT:-}" ]; then + echo "PARQUET_N6_JAVA_JDK_ROOT is required" >&2 + exit 2 +fi +if [ -z "${PARQUET_N6_JAVA_ROOT:-}" ]; then + echo "PARQUET_N6_JAVA_ROOT is required" >&2 + exit 2 +fi +python="$PARQUET_N6_INTEROP_PYTHON_ROOT/bin/python3.12" +if [ ! -x "$python" ]; then + echo "the pinned Python interpreter is not executable: $python" >&2 + exit 2 +fi +corpus_root=$1 +raw_evidence=$2 +output=$3 +shift 3 + +exec "$python" -I -S -B "$oracle_dir/run.py" \ + --repository "$repository" \ + --manifest "$repository/test/conformance/n6/manifest.toml" \ + --capabilities "$repository/test/conformance/n6/capabilities.toml" \ + --fixtures "$repository/test/conformance/n6/fixtures.toml" \ + --descriptor "$oracle_dir/toolchain.toml" \ + --raw-evidence "$raw_evidence" \ + --corpus-root "$corpus_root" \ + --java-root "$PARQUET_N6_JAVA_ROOT" \ + --jdk-root "$PARQUET_N6_JAVA_JDK_ROOT" \ + --output "$output" \ + "$@" diff --git a/test/conformance/n6/oracles/parquet-java/runtests.py b/test/conformance/n6/oracles/parquet-java/runtests.py new file mode 100755 index 0000000..0b2f183 --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/runtests.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +import argparse +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile +import tomllib +import types +import unittest + +SCRIPT_DIRECTORY = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIRECTORY)) +import run as harness +sys.path.pop(0) + + +EXPECTED_LOGICAL_DIGESTS = { + "apache-alltypes-dictionary": + "f2c929bbd05fdfba22f2c0b2099cda5abee2736540c0e3c08c35c2ccfe7faa64", + "apache-alltypes-plain": + "7e7fe74a6cbcee312b5d69c37118b1ddee5e012dd0ec2cacc94367f732c611d3", + "apache-binary": + "abbae1d98f07cc72a89cc0d6cb3f2c062139148f320091ab34f82116608fa603", + "apache-bson": + "eeb323c79a61256a0dc724c3cb527e5f95e11c38c24b66d98a345e977342458a", + "apache-byte-array-decimal": + "72d58c34be4872a9511f6aa90e7aa85da7cd5f6b74bf94670c25e5e103f91345", + "apache-fixed-length-byte-array": + "b774417b00a0769e81247e81a13ba8ae2235655ff7fa7d19c5c63b0189b29f43", + "apache-fixed-length-decimal": + "695d04714cd772edd99e27dadc1c48e21c0ba50c6905641b8d0e26294813bb24", + "apache-fixed-length-decimal-legacy": + "ce517a369e588bbacded5d71385f357d5135664b0cf4c664609f1b9a4eaaebfb", + "apache-int32-decimal": + "457098e384bc3398752bc6c584730a4f66a8b589f124df2049538509b74f8788", + "apache-int32-with-null-pages": + "40ad3a2c665a0cf20c7be47b8468874831baa3294275ae3b5698991cb15af5c7", + "apache-int64-decimal": + "995659f0eae712e82d5a40cf15a07743550708c97b6c30630a8415492f80ebab", + "apache-json": + "34cad7fad382e26359ad83599fcf1d58d1528fcfb2f6b05f648d1da4ded933a9", + "apache-rle-boolean-encoding": + "898e936d35669810ecbc7f2cb85ed6885cc769c07d8157292b93936dddf743fa", +} +EXPECTED_TYPE_ORDER_DIGESTS = { + "apache-binary": + "398ef274d55a8ecbc60a45dad6d53767ff64690f02041ad62fbe18d4d8ade9b2", + "apache-binary-truncated-min-max": + "386d4867bea929fc5421a102d4ad9f94d279622822fb64ea7ea28f72ec07abae", + "apache-bson": + "5064c7fe7c81fcfa0c01a590d864b9fd332c724bd4bddc90a2ca07b8492ace7c", + "apache-fixed-length-byte-array": + "a31a068e65e43cac96ee043dfdd836b8d412001c4bea16472579e2dc00e787af", + "apache-float16-nonzeros-and-nans": + "495cb01fc4407b043081b8e7815151eb93a56a576cdce8ede97bbb262c5c444b", + "apache-float16-zeros-and-nans": + "1199284c5ebe019e4f99695de404506d50d612fce387e040b9bc48d995b69665", + "apache-int32-with-null-pages": + "febe5e5ab7300d06cee411673b296e372b115b7834ddf23dd03b2235ec8d275a", + "apache-json": + "f92c2356f5af0856938651d50f667dead0b5e510f4137b66ce4a53dc83d89a04", + "apache-nan-in-stats": + "3c3ec0157bc53c7e83c4f2c86357037471e104f67d1c1db3024da46d6780b3c7", + "apache-single-nan": + "6467555af73803cc0ea6d17fe2a2f03948c40dd48549bdf73eb5da2baec22c2d", +} +EXPECTED_LEGACY_DIGESTS = { + "apache-fixed-length-decimal": + "ed5bdd8b71b0b625d793eb846f962ab257e36d1611070b4a33b2a916ec3f9e4d", + "apache-fixed-length-decimal-legacy": + "acd3c50a51cf263624be52f9e43beb2ce75939f6a9991a91084ef425df321ecb", + "apache-int32-decimal": + "d0ff4a81cfe9b6d19fdb0400828f2749288515bfc1e9a8b02a4b137c998608d5", + "apache-int64-decimal": + "c7187495c163ebd92ad33036367bb9d88ffb6211e1340e42f982c3195d0d738d", +} +EXPECTED_POLICY_DIGEST = \ + "cdaa409cc6dada3fada0752543aa6911174cd2523a38f996490e814bc1f58b1a" +EXPECTED_POLICY_FACTS = ( + ("null-binary", None, "BINARY", "not_applicable", "error", None, + None, None, True), + ("empty-binary", "", "BINARY", "not_applicable", "error", None, + None, None, True), + ("empty-application-binary", " version 1.0.0", "BINARY", + "not_applicable", "error", None, None, None, True), + ("unparsable-binary", "garbage!", "BINARY", "not_applicable", + "error", None, None, None, True), + ("unrelated-binary", "impala version 1.0.0", "BINARY", + "not_applicable", "parsed", "impala", "1.0.0", None, False), + ("missing-semver-binary", "parquet-mr version", "BINARY", + "not_applicable", "parsed", "parquet-mr", None, None, True), + ("before-fix-distinct-binary", "parquet-mr version 1.7.9", "BINARY", + "distinct", "parsed", "parquet-mr", "1.7.9", None, True), + ("before-fix-equal-binary", "parquet-mr version 1.7.9", "BINARY", + "equal", "parsed", "parquet-mr", "1.7.9", None, True), + ("release-candidate-binary", "parquet-mr version 1.8.0-rc1", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.8.0-rc1", None, + True), + ("fixed-binary", "parquet-mr version 1.8.0", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.8.0", None, False), + ("cdh-before-binary", "parquet-mr version 1.5.0-cdh5.4.9", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.5.0-cdh5.4.9", + None, True), + ("cdh-start-binary", "parquet-mr version 1.5.0-cdh5.5.0", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.5.0-cdh5.5.0", + None, False), + ("cdh-end-binary", "parquet-mr version 1.5.0", "BINARY", + "not_applicable", "parsed", "parquet-mr", "1.5.0", None, True), + ("before-fix-fixed", "parquet-mr version 1.7.9", + "FIXED_LEN_BYTE_ARRAY", "not_applicable", "parsed", "parquet-mr", + "1.7.9", None, True), + ("fixed-fixed", "parquet-mr version 1.8.0", + "FIXED_LEN_BYTE_ARRAY", "not_applicable", "parsed", "parquet-mr", + "1.8.0", None, False), + ("before-fix-int32", "parquet-mr version 1.7.9", "INT32", + "not_applicable", "parsed", "parquet-mr", "1.7.9", None, False), +) + + +def leaf(physical, logical="NONE", **fields): + schema = { + "physical_type": physical, + "logical_type": logical, + "converted_type": None, + "type_length": None, + "precision": None, + "scale": None, + "bit_width": None, + "is_signed": None, + "time_unit": None, + "is_adjusted_to_utc": None, + "crs": None, + "geography_algorithm": None, + } + schema.update(fields) + return {"leaf_schema": schema, "path": ["value"]} + + +class ParquetJavaHarnessTests(unittest.TestCase): + def test_canonical_values_preserve_bits_bytes_and_decimals(self): + decimal_leaf = leaf("INT32", "DECIMAL", converted_type="DECIMAL", + precision=4, scale=2) + self.assertEqual(harness.canonical_value( + {"kind": "int32", "value": "-125"}, decimal_leaf), + {"decimal_scale": 2, "unscaled": "-125"}) + binary_decimal = leaf("BYTE_ARRAY", "DECIMAL", + converted_type="DECIMAL", precision=4, scale=2) + self.assertEqual(harness.canonical_value( + {"kind": "binary", "value": "ff83"}, binary_decimal), + {"decimal_scale": 2, "unscaled": "-125"}) + self.assertEqual(harness.canonical_value( + {"kind": "float32_bits", "value": "80000000"}, + leaf("FLOAT")), {"float32_bits": "80000000"}) + self.assertEqual(harness.canonical_value( + {"kind": "binary", "value": "00ff"}, leaf("BYTE_ARRAY")), + {"bytes_hex": "00ff"}) + + def test_int96_conversion_is_exact(self): + value = {"kind": "int96", "value": + "01000000000000008c3d2500"} + self.assertEqual(harness.canonical_value(value, leaf("INT96")), + {"timestamp_nanoseconds": "1"}) + invalid = {"kind": "int96", "value": + "00004f91944e00008c3d2500"} + with self.assertRaises(harness.HarnessError): + harness.canonical_value(invalid, leaf("INT96")) + + def test_java_command_has_fixed_limits_and_no_build_tool(self): + toolchain = { + "harness": pathlib.Path("/inputs/harness.jar"), + "parquet-cli-runtime": pathlib.Path("/inputs/parquet.jar"), + "hadoop-client-api": pathlib.Path("/inputs/hadoop-api.jar"), + "hadoop-client-runtime": pathlib.Path("/inputs/hadoop-runtime.jar"), + } + command = harness.java_command(pathlib.Path("/jdk"), toolchain, + "audit", pathlib.Path("/inputs/file.parquet")) + self.assertEqual(command[0], "/jdk/bin/java") + self.assertIn("-Xmx256m", command) + self.assertIn("-XX:MaxMetaspaceSize=128m", command) + self.assertNotIn("mvn", " ".join(command).lower()) + self.assertNotIn("http", " ".join(command).lower()) + self.assertEqual(command[-2:], ["audit", "/inputs/file.parquet"]) + + def test_java_environment_drops_ambient_injection_variables(self): + environment = harness.java_environment(pathlib.Path("/jdk")) + self.assertEqual(set(environment), + {"JAVA_HOME", "LANG", "LC_ALL", "TZ"}) + self.assertNotIn("CLASSPATH", environment) + self.assertNotIn("JAVA_TOOL_OPTIONS", environment) + + def test_bounded_process_caps_stdout_and_rejects_failure(self): + environment = {"PATH": "/usr/bin:/bin"} + cwd = pathlib.Path.cwd() + stdout, stderr = harness.run_bounded( + [sys.executable, "-c", "print('ok')"], environment, cwd, + 16, 16, 5) + self.assertEqual(stdout, b"ok\n") + self.assertEqual(stderr, b"") + with self.assertRaises(harness.HarnessError): + harness.run_bounded([sys.executable, "-c", + "import os; os.write(1, b'x' * 1024)"], environment, cwd, + 16, 16, 5) + with self.assertRaises(harness.HarnessError): + harness.run_bounded([sys.executable, "-c", + "raise SystemExit(3)"], environment, cwd, 16, 1024, 5) + + def test_input_and_output_reject_symbolic_links(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory).resolve(strict=True) + target = root / "target" + target.mkdir() + source = target / "input.parquet" + source.write_bytes(b"PAR1") + linked = root / "linked" + linked.symlink_to(target, target_is_directory=True) + with self.assertRaises(harness.HarnessError): + harness.checked_regular_file(linked / source.name, "input") + with self.assertRaises(harness.HarnessError): + harness.checked_output_path(linked / "output.jsonl") + + def test_claim_scope_is_exact(self): + claims = {(case_id, "read.logical-values"): "planned" + for case_id in harness.EXPECTED_LOGICAL_CASES} + claims.update({(case_id, "compat.legacy-statistics"): "planned" + for case_id in harness.EXPECTED_LEGACY_CASES}) + claims.update({key: "unsupported" + for key in harness.EXPECTED_UNSUPPORTED}) + self.assertIsNone(harness.validate_claim_scope(claims)) + claims.update({(case_id, "wire.column-order.type"): "planned" + for case_id in harness.EXPECTED_TYPE_ORDER_CASES}) + self.assertIsNone(harness.validate_claim_scope(claims)) + del claims[(next(iter(harness.EXPECTED_TYPE_ORDER_CASES)), + "wire.column-order.type")] + with self.assertRaises(harness.HarnessError): + harness.validate_claim_scope(claims) + claims = {(case_id, "read.logical-values"): "planned" + for case_id in harness.EXPECTED_LOGICAL_CASES} + claims.update({(case_id, "compat.legacy-statistics"): "planned" + for case_id in harness.EXPECTED_LEGACY_CASES}) + claims.update({key: "unsupported" + for key in harness.EXPECTED_UNSUPPORTED}) + claims[("extra", "read.logical-values")] = "planned" + with self.assertRaises(harness.HarnessError): + harness.validate_claim_scope(claims) + + def test_type_order_observations_are_normalized(self): + raw_file = {"case_id": "case", "record": "file"} + raw_column = { + "case_id": "case", + "column_order": {"state": "TYPE_ORDER"}, + "leaf": 0, + "num_values": "3", + "path": ["value"], + "leaf_schema": {"physical_type": "BYTE_ARRAY"}, + "record": "column_statistics", + "row_group": 0, + } + document = { + "schema": [{"column_order": "TYPE_DEFINED_ORDER"}], + "row_groups": [{ + "columns": [{ + "has_non_null_value": True, + "max_hex": "ff", + "min_hex": "00", + "num_nulls": 1, + "path": ["value"], + "physical_type": "BINARY", + }], + "row_group": 0, + "row_count": 3, + }], + } + raw_columns = {("case", 0, 0): raw_column} + expected = [{"file": raw_file, "columns": [raw_column]}] + self.assertEqual(harness.type_order_observations(document, "case", + raw_file, raw_columns), expected) + document["schema"][0]["column_order"] = "UNDEFINED" + with self.assertRaises(harness.HarnessError): + harness.type_order_observations(document, "case", raw_file, + raw_columns) + + def test_build_rejects_unsafe_write_paths_before_inputs(self): + cases = ( + ("build-link", "build root must be a real directory"), + ("build-file", "build root must be a real directory"), + ("artifacts-link", "artifact directory must be a real directory"), + ("artifacts-file", "artifact directory must be a real directory"), + ("output-link", "artifact output must be a regular non-link file"), + ("output-directory", + "artifact output must be a regular non-link file"), + ) + for scenario, expected in cases: + with self.subTest(scenario=scenario), \ + tempfile.TemporaryDirectory() as directory: + oracle = pathlib.Path(directory) / "oracle" + oracle.mkdir() + script = oracle / "build.sh" + shutil.copyfile(SCRIPT_DIRECTORY / "build.sh", script) + target = pathlib.Path(directory) / "target" + target.mkdir() + build = oracle / "build" + if scenario == "build-link": + build.symlink_to(target, target_is_directory=True) + elif scenario == "build-file": + build.write_text("not a directory") + else: + build.mkdir() + artifacts = build / "artifacts" + if scenario == "artifacts-link": + artifacts.symlink_to(target, target_is_directory=True) + elif scenario == "artifacts-file": + artifacts.write_text("not a directory") + else: + artifacts.mkdir() + output = artifacts / \ + "parquet-java-n6-harness.jar" + if scenario == "output-link": + output.symlink_to(target / "artifact.jar") + else: + output.mkdir() + result = subprocess.run(["/bin/sh", str(script)], + check=False, capture_output=True, text=True, + env={"PATH": "/usr/bin:/bin"}) + self.assertEqual(result.returncode, 1) + self.assertIn(expected, result.stderr) + + def test_policy_observations_and_digest_are_frozen(self): + self.assertEqual(harness.EXPECTED_POLICY_FACTS, + EXPECTED_POLICY_FACTS) + observations = harness.expected_policy_observations() + self.assertEqual(len(observations), 16) + self.assertEqual( + harness.common.observation_digest(harness.POLICY_CASE, + "compat.legacy-statistics", observations), + EXPECTED_POLICY_DIGEST) + + def test_descriptor_pins_current_wrapper_sources(self): + repository = SCRIPT_DIRECTORY.parents[4] + with (SCRIPT_DIRECTORY / "toolchain.toml").open("rb") as stream: + descriptor = tomllib.load(stream) + authority = { + "version": "1.17.1", + "revision": "78a8d3230eb4769db93de5f2f2e18363c04cae81", + } + validated = harness.validate_descriptor(repository, descriptor, + authority) + self.assertEqual(validated["producer"], "parquet-java") + self.assertEqual(len(validated["source"]), 9) + self.assertEqual(len(validated["wrapper"]), 7) + + +def integration_arguments(arguments, output, check): + repository = pathlib.Path(arguments.repository).resolve(strict=True) + return types.SimpleNamespace( + repository=str(repository), + manifest=str(repository / "test/conformance/n6/manifest.toml"), + capabilities=str(repository / "test/conformance/n6/capabilities.toml"), + fixtures=str(repository / "test/conformance/n6/fixtures.toml"), + descriptor=str(SCRIPT_DIRECTORY / "toolchain.toml"), + raw_evidence=arguments.raw_evidence, + corpus_root=arguments.corpus_root, + java_root=arguments.java_root, + jdk_root=arguments.jdk_root, + output=str(output), + check=check, + draft=arguments.draft, + ) + + +def validate_integration_records(path): + records = [json.loads(line) for line in path.read_text().splitlines()] + if len(records) != harness.EXPECTED_RECORD_COUNT: + raise AssertionError( + f"expected {harness.EXPECTED_RECORD_COUNT} records, got {len(records)}") + results = {(record["case_id"], record["capability_id"]): record + for record in records if record["record"] == "case_result"} + logical = {case_id: results[(case_id, + "read.logical-values")]["actual_sha256"] + for case_id in harness.EXPECTED_LOGICAL_CASES} + if logical != EXPECTED_LOGICAL_DIGESTS: + raise AssertionError("Parquet Java logical digests differ") + type_order = {case_id: results[(case_id, + "wire.column-order.type")]["actual_sha256"] + for case_id in harness.EXPECTED_TYPE_ORDER_CASES} + if type_order != EXPECTED_TYPE_ORDER_DIGESTS: + raise AssertionError("Parquet Java TYPE_ORDER digests differ") + legacy = {case_id: results[(case_id, + "compat.legacy-statistics")]["actual_sha256"] + for case_id in EXPECTED_LEGACY_DIGESTS} + if legacy != EXPECTED_LEGACY_DIGESTS: + raise AssertionError("Parquet Java legacy digests differ") + policy = results[(harness.POLICY_CASE, + "compat.legacy-statistics")]["actual_sha256"] + if policy != EXPECTED_POLICY_DIGEST: + raise AssertionError("Parquet Java policy digest differs") + unsupported = {key for key, record in results.items() + if record["status"] == "UNSUPPORTED"} + if unsupported != harness.EXPECTED_UNSUPPORTED: + raise AssertionError("Parquet Java unsupported results differ") + if len({record["case_id"] for record in records + if record["record"] == "file"}) != 19: + raise AssertionError("Parquet Java file coverage differs") + if sum(record["status"] == "PASS" for record in results.values()) != 28: + raise AssertionError("Parquet Java PASS count differs") + return None + + +def run_integration(arguments): + repository = pathlib.Path(arguments.repository).resolve(strict=True) + output = repository / \ + "test/conformance/n6/evidence/parquet-java.normalized.jsonl" + harness.generate(integration_arguments(arguments, output, False)) + first = output.read_bytes() + validate_integration_records(output) + harness.generate(integration_arguments(arguments, output, True)) + if output.read_bytes() != first: + raise AssertionError("Parquet Java check mode changed the output") + print(f"Parquet Java integration passed: " + f"{harness.EXPECTED_RECORD_COUNT} records, {len(first)} bytes") + return None + + +def parser(): + result = argparse.ArgumentParser() + result.add_argument("--integration", action="store_true") + result.add_argument("--repository") + result.add_argument("--corpus-root") + result.add_argument("--raw-evidence") + result.add_argument("--java-root") + result.add_argument("--jdk-root") + result.add_argument("--draft", action="store_true") + return result + + +def main(): + arguments = parser().parse_args() + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + ParquetJavaHarnessTests) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if not result.wasSuccessful(): + return 1 + if arguments.integration: + required = (arguments.repository, arguments.corpus_root, + arguments.raw_evidence, arguments.java_root, arguments.jdk_root) + if any(value is None for value in required): + raise SystemExit("--integration requires --repository, " + "--corpus-root, --raw-evidence, --java-root, and --jdk-root") + run_integration(arguments) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/conformance/n6/oracles/parquet-java/src/org/julialang/parquet/n6/java/AuditMain.java b/test/conformance/n6/oracles/parquet-java/src/org/julialang/parquet/n6/java/AuditMain.java new file mode 100644 index 0000000..7cb42ec --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/src/org/julialang/parquet/n6/java/AuditMain.java @@ -0,0 +1,326 @@ +package org.julialang.parquet.n6.java; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.CorruptStatistics; +import org.apache.parquet.VersionParser; +import org.apache.parquet.VersionParser.ParsedVersion; +import org.apache.parquet.VersionParser.VersionParseException; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.statistics.Statistics; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.api.ReadSupport; +import org.apache.parquet.hadoop.example.GroupReadSupport; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.InputFile; +import org.apache.parquet.io.LocalInputFile; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; + +public final class AuditMain { + private static final String PRODUCER = "parquet-java"; + private static final String PRODUCER_VERSION = "1.17.1"; + private static final String SOURCE_REVISION = "78a8d3230eb4769db93de5f2f2e18363c04cae81"; + private static final int MAX_COLUMNS = 1024; + private static final int MAX_ROWS = 1_000_000; + private static final int MAX_BINARY_BYTES = 16 * 1024 * 1024; + private static final ObjectMapper JSON = new ObjectMapper(); + + private AuditMain() {} + + public static void main(String[] arguments) throws Exception { + if (arguments.length == 2 && "audit".equals(arguments[0])) { + JSON.writeValue(System.out, audit(checkedInput(arguments[1]))); + System.out.write('\n'); + return; + } + if (arguments.length == 1 && "policy".equals(arguments[0])) { + JSON.writeValue(System.out, policy()); + System.out.write('\n'); + return; + } + throw new IllegalArgumentException("usage: AuditMain audit FILE | policy"); + } + + private static Path checkedInput(String value) throws IOException { + Path path = Paths.get(value).toAbsolutePath().normalize(); + if (Files.isSymbolicLink(path) + || !Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("audit input is not a regular non-link file"); + } + return path; + } + + private static Map oracle() { + Map result = new LinkedHashMap<>(); + result.put("name", PRODUCER); + result.put("version", PRODUCER_VERSION); + result.put("commit", SOURCE_REVISION); + return result; + } + + private static Map audit(Path path) throws IOException { + InputFile input = new LocalInputFile(path); + ParquetMetadata footer; + try (ParquetFileReader reader = ParquetFileReader.open(input)) { + footer = reader.getFooter(); + } + MessageType schema = footer.getFileMetaData().getSchema(); + List columns = schema.getColumns(); + if (columns.size() > MAX_COLUMNS) { + throw new IOException("column count exceeds the audit limit"); + } + requireFlat(columns); + List>> values = emptyColumns(columns.size()); + long rowCount = readRows(path, columns, values); + Map result = new LinkedHashMap<>(); + result.put("oracle", oracle()); + result.put("action", "audit"); + result.put("created_by", footer.getFileMetaData().getCreatedBy()); + result.put("schema", schemaRecords(columns)); + result.put("row_count", rowCount); + result.put("columns", values); + result.put("row_groups", statisticsRecords(footer)); + return result; + } + + private static void requireFlat(List columns) throws IOException { + for (ColumnDescriptor column : columns) { + if (column.getPath().length != 1 || column.getMaxRepetitionLevel() != 0) { + throw new IOException("the reviewed logical-value fixture is not flat"); + } + } + } + + private static List>> emptyColumns(int count) { + List>> result = new ArrayList<>(count); + for (int index = 0; index < count; index += 1) { + result.add(new ArrayList<>()); + } + return result; + } + + private static long readRows( + Path path, + List columns, + List>> values) throws IOException { + long rowCount = 0; + try (ParquetReader reader = + new LocalGroupReaderBuilder(new LocalInputFile(path)).build()) { + Group row; + while ((row = reader.read()) != null) { + if (rowCount >= MAX_ROWS) { + throw new IOException("row count exceeds the audit limit"); + } + for (int index = 0; index < columns.size(); index += 1) { + int count = row.getFieldRepetitionCount(index); + if (count > 1) { + throw new IOException("the reviewed logical-value fixture has repeated values"); + } + values.get(index).add(count == 0 ? null : encodedValue(row, index, columns.get(index))); + } + rowCount += 1; + } + } + return rowCount; + } + + private static Map encodedValue( + Group row, int index, ColumnDescriptor column) throws IOException { + PrimitiveTypeName type = column.getPrimitiveType().getPrimitiveTypeName(); + switch (type) { + case BOOLEAN: + return tagged("boolean", row.getBoolean(index, 0)); + case INT32: + return tagged("int32", Integer.toString(row.getInteger(index, 0))); + case INT64: + return tagged("int64", Long.toString(row.getLong(index, 0))); + case FLOAT: + return tagged( + "float32_bits", + String.format("%08x", Float.floatToRawIntBits(row.getFloat(index, 0)))); + case DOUBLE: + return tagged( + "float64_bits", + String.format("%016x", Double.doubleToRawLongBits(row.getDouble(index, 0)))); + case BINARY: + return taggedBinary("binary", row.getBinary(index, 0)); + case FIXED_LEN_BYTE_ARRAY: + return taggedBinary("fixed", row.getBinary(index, 0)); + case INT96: + return taggedBinary("int96", row.getInt96(index, 0)); + default: + throw new IOException("unsupported physical type: " + type); + } + } + + private static Map tagged(String kind, Object value) { + Map result = new LinkedHashMap<>(); + result.put("kind", kind); + result.put("value", value); + return result; + } + + private static Map taggedBinary(String kind, Binary value) throws IOException { + byte[] bytes = value.getBytes(); + if (bytes.length > MAX_BINARY_BYTES) { + throw new IOException("binary value exceeds the audit limit"); + } + return tagged(kind, hex(bytes)); + } + + private static List> schemaRecords(List columns) { + List> result = new ArrayList<>(columns.size()); + for (ColumnDescriptor column : columns) { + Map record = new LinkedHashMap<>(); + record.put("name", column.getPath()[0]); + record.put("physical_type", column.getPrimitiveType().getPrimitiveTypeName().name()); + record.put("maximum_definition_level", column.getMaxDefinitionLevel()); + record.put("maximum_repetition_level", column.getMaxRepetitionLevel()); + record.put( + "column_order", + column.getPrimitiveType().columnOrder().getColumnOrderName().name()); + result.add(record); + } + return result; + } + + private static List> statisticsRecords(ParquetMetadata footer) { + List> groups = new ArrayList<>(); + int ordinal = 0; + for (BlockMetaData block : footer.getBlocks()) { + Map group = new LinkedHashMap<>(); + group.put("row_group", ordinal); + group.put("row_count", block.getRowCount()); + List> columns = new ArrayList<>(); + for (ColumnChunkMetaData column : block.getColumns()) { + Statistics statistics = column.getStatistics(); + Map record = new LinkedHashMap<>(); + record.put("path", Arrays.asList(column.getPath().toArray())); + record.put("physical_type", column.getType().name()); + record.put("statistics_class", statistics.getClass().getName()); + record.put("empty", statistics.isEmpty()); + record.put("has_non_null_value", statistics.hasNonNullValue()); + record.put("num_nulls_set", statistics.isNumNullsSet()); + record.put("num_nulls", statistics.isNumNullsSet() ? statistics.getNumNulls() : null); + record.put("min_hex", statistics.hasNonNullValue() ? hex(statistics.getMinBytes()) : null); + record.put("max_hex", statistics.hasNonNullValue() ? hex(statistics.getMaxBytes()) : null); + record.put( + "should_ignore_statistics", + CorruptStatistics.shouldIgnoreStatistics( + footer.getFileMetaData().getCreatedBy(), column.getType())); + columns.add(record); + } + group.put("columns", columns); + groups.add(group); + ordinal += 1; + } + return groups; + } + + private static Map policy() { + List> observations = new ArrayList<>(); + observations.add(policyCase("null-binary", null, PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase("empty-binary", "", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "empty-application-binary", " version 1.0.0", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "unparsable-binary", "garbage!", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "unrelated-binary", "impala version 1.0.0", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "missing-semver-binary", "parquet-mr version", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "before-fix-distinct-binary", "parquet-mr version 1.7.9", PrimitiveTypeName.BINARY, "distinct")); + observations.add(policyCase( + "before-fix-equal-binary", "parquet-mr version 1.7.9", PrimitiveTypeName.BINARY, "equal")); + observations.add(policyCase( + "release-candidate-binary", "parquet-mr version 1.8.0-rc1", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "fixed-binary", "parquet-mr version 1.8.0", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "cdh-before-binary", "parquet-mr version 1.5.0-cdh5.4.9", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "cdh-start-binary", "parquet-mr version 1.5.0-cdh5.5.0", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "cdh-end-binary", "parquet-mr version 1.5.0", PrimitiveTypeName.BINARY, "not_applicable")); + observations.add(policyCase( + "before-fix-fixed", "parquet-mr version 1.7.9", PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, + "not_applicable")); + observations.add(policyCase( + "fixed-fixed", "parquet-mr version 1.8.0", PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, + "not_applicable")); + observations.add(policyCase( + "before-fix-int32", "parquet-mr version 1.7.9", PrimitiveTypeName.INT32, "not_applicable")); + Map result = new LinkedHashMap<>(); + result.put("oracle", oracle()); + result.put("action", "policy"); + result.put("observations", observations); + return result; + } + + private static Map policyCase( + String scenarioId, + String createdBy, + PrimitiveTypeName physicalType, + String boundRelation) { + Map result = new LinkedHashMap<>(); + result.put("scenario_id", scenarioId); + result.put("created_by", createdBy); + result.put("physical_type", physicalType.name()); + result.put("bound_relation", boundRelation); + result.put("version_parse", versionParse(createdBy)); + result.put("should_ignore_statistics", CorruptStatistics.shouldIgnoreStatistics(createdBy, physicalType)); + return result; + } + + private static Map versionParse(String createdBy) { + Map result = new LinkedHashMap<>(); + try { + ParsedVersion parsed = VersionParser.parse(createdBy); + result.put("status", "parsed"); + result.put("application", parsed.application); + result.put("version", parsed.version); + result.put("build", parsed.appBuildHash); + } catch (RuntimeException | VersionParseException error) { + result.put("status", "error"); + } + return result; + } + + private static String hex(byte[] bytes) { + char[] digits = "0123456789abcdef".toCharArray(); + char[] result = new char[bytes.length * 2]; + for (int index = 0; index < bytes.length; index += 1) { + int value = bytes[index] & 0xff; + result[index * 2] = digits[value >>> 4]; + result[index * 2 + 1] = digits[value & 0x0f]; + } + return new String(result); + } + + private static final class LocalGroupReaderBuilder extends ParquetReader.Builder { + private LocalGroupReaderBuilder(InputFile input) { + super(input); + } + + @Override + protected ReadSupport getReadSupport() { + return new GroupReadSupport(); + } + } +} diff --git a/test/conformance/n6/oracles/parquet-java/toolchain.toml b/test/conformance/n6/oracles/parquet-java/toolchain.toml new file mode 100644 index 0000000..1a980c9 --- /dev/null +++ b/test/conformance/n6/oracles/parquet-java/toolchain.toml @@ -0,0 +1,108 @@ +descriptor_version = 1 +status = "verified" +producer = "parquet-java" +producer_version = "1.17.1" +source_revision = "78a8d3230eb4769db93de5f2f2e18363c04cae81" +platform = "macos-15-arm64" +python_version = "3.12.8" +python_distribution_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250115/cpython-3.12.8%2B20250115-aarch64-apple-darwin-install_only_stripped.tar.gz" +python_distribution_sha256 = "dfb8a4c87116538717105ef3dec3668ae07590a5b5532109fec3ccad90be2fbc" +python_tree_policy = "extract-strip-site-packages-bytecode-v1" +python_executable_sha256 = "d6b64f766d3b08326aa10cdb37c9d922e3af38b57ec07caa28b894e5fccf6e69" +python_tree_sha256 = "e3b7dcdffba67f605b0fa3318656387e8d4265d34531c2bdbc43d3aba4a033ec" +java_vendor = "Eclipse-Adoptium-Temurin" +java_version = "21.0.8+9" +java_executable_sha256 = "0045ae168ee132bbf469a26fb17dac6d1dee431c9b7826474f3b6ee574a997c9" +javac_executable_sha256 = "7be7937fc6bae0ca89f0866f9ce94fc40a935dfb87806d3c701eca3402cfb90a" +java_release_sha256 = "8e98b265f9a6fd3db04d2535108497897e87f1b3821270cf10ffa937463dc2ee" +jdk_tree_sha256 = "7ffead12e2614843ffcc77e36d1bd142b921d91381d4ef12827c32c54369d209" +hadoop_version = "3.3.0" +harness_main = "org.julialang.parquet.n6.java.AuditMain" +harness_jar_file = "test/conformance/n6/oracles/parquet-java/build/artifacts/parquet-java-n6-harness.jar" +harness_jar_sha256 = "7d6de1067e4e01de65f5868c8643f4a68fe4ff6afc50759716d685bd7b9e764c" +harness_jar_size = 9408 + +[[artifact]] +id = "parquet-cli-runtime" +file = "test/conformance/n6/oracles/parquet-java/build/artifacts/parquet-cli-1.17.1-runtime.jar" +url = "https://repo.maven.apache.org/maven2/org/apache/parquet/parquet-cli/1.17.1/parquet-cli-1.17.1-runtime.jar" +sha256 = "d0173051493c506a298c691e555a41a682a405895fd0c8cc429a7e1cb1fcc711" +size = 50072122 + +[[artifact]] +id = "hadoop-client-api" +file = "test/conformance/n6/oracles/parquet-java/build/artifacts/hadoop-client-api-3.3.0.jar" +url = "https://repo.maven.apache.org/maven2/org/apache/hadoop/hadoop-client-api/3.3.0/hadoop-client-api-3.3.0.jar" +sha256 = "d549ba6d131fd6c8e5d42a78dab5c790950edd6258523dedc556b537ca6654aa" +size = 19207034 + +[[artifact]] +id = "hadoop-client-runtime" +file = "test/conformance/n6/oracles/parquet-java/build/artifacts/hadoop-client-runtime-3.3.0.jar" +url = "https://repo.maven.apache.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.0/hadoop-client-runtime-3.3.0.jar" +sha256 = "2ba23f1e1dbb03e73600a41fcb187ad2626529684ed226085a91b0a0d6d67ee5" +size = 27255121 + +[[source]] +file = "parquet-common/src/main/java/org/apache/parquet/VersionParser.java" +sha256 = "a8c54497632bbfacd2da1d52fcf83e3ee6c7e4c0134048c4649bd50055866c87" + +[[source]] +file = "parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java" +sha256 = "1344e06d6644c9b8d027d03d3e8a595fc60f029eef5e6156b63248a4ee346dc7" + +[[source]] +file = "parquet-common/src/main/java/org/apache/parquet/io/LocalInputFile.java" +sha256 = "3112f6c7e9f5eb6f0c90d15638f8c7e3c20f004cfc9dd1e9572801453aebcad6" + +[[source]] +file = "parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java" +sha256 = "258cb4a4b5f6b6a846c51ebf13d868fe4907ee3c8dfa6e5e73484e927564d1cd" + +[[source]] +file = "parquet-column/src/main/java/org/apache/parquet/example/data/Group.java" +sha256 = "12cda259d85654fa49d195b948b0faddbb9229f070e0cdda6521d42b3e282907" + +[[source]] +file = "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java" +sha256 = "ce64a167e397014f475c4538692cb6c4ce93331a976af78dec4a204803e6eb0e" + +[[source]] +file = "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java" +sha256 = "85f46c06cf3d1b65512031fcb850879372533e377680c2a9389dd421257b63d8" + +[[source]] +file = "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/example/GroupReadSupport.java" +sha256 = "71a18e14d2bc8afd78c817f2456a9f77a34b1b5e176156bc58bdd857ecdfd535" + +[[source]] +file = "parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java" +sha256 = "45ce3aa85aaca19f9aa0195be6b6a490994d3747d1c4160fb05d15be3db9998b" + +[[wrapper]] +path = "test/conformance/n6/harnesses/common.py" +sha256 = "8b48613a795deed37089f6dbfca8eb6ed18062ebfd907a4606c91ef3308776ad" + +[[wrapper]] +path = "test/conformance/n6/oracles/parquet-java/build.sh" +sha256 = "26023ade8fc0bd16433c2ac91ae8a64285e7c8fac68f8f818a7e788a63859973" + +[[wrapper]] +path = "test/conformance/n6/oracles/parquet-java/check.sh" +sha256 = "1b7a43e1d9c57784551fbc9decad07581a0cd9282a09939b457007cf83ad65e3" + +[[wrapper]] +path = "test/conformance/n6/oracles/parquet-java/run.py" +sha256 = "8cd6aa7e8efadf11a916e549330a6b3bd44a025d53f8cc55c3ecc82c15dcc85d" + +[[wrapper]] +path = "test/conformance/n6/oracles/parquet-java/run.sh" +sha256 = "68e2573a1636ca5424f3418f80221bd8af4fc660a985d58acbc1c464f197cf92" + +[[wrapper]] +path = "test/conformance/n6/oracles/parquet-java/runtests.py" +sha256 = "9f671343988d078752140f231cb81a8ea01b24fa164cb0a3671770e13361cd0d" + +[[wrapper]] +path = "test/conformance/n6/oracles/parquet-java/src/org/julialang/parquet/n6/java/AuditMain.java" +sha256 = "8be45f063510bbd04f95115e8572eb4cff40914b35f5f2334452edb3f423e60a" diff --git a/test/conformance/n6/oracles/raw-java/.gitignore b/test/conformance/n6/oracles/raw-java/.gitignore new file mode 100644 index 0000000..495a75e --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/.gitignore @@ -0,0 +1,2 @@ +/build/ + diff --git a/test/conformance/n6/oracles/raw-java/README.md b/test/conformance/n6/oracles/raw-java/README.md new file mode 100644 index 0000000..59cf319 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/README.md @@ -0,0 +1,163 @@ +# Raw Parquet 2.13 footer scanner + +This directory contains a test-only Java scanner for raw Parquet footer +evidence. It generates Java format classes from the pinned Parquet 2.13 IDL. +It then decodes Compact Thrift in a separate JVM process. + +This scanner does not use Parquet.jl, parquet-java, or parquet-java's embedded +Parquet 2.12 format classes. It does not prove parquet-java semantic support. +It records raw Parquet 2.13 wire evidence only. + +## Authority and fixed inputs + +The N6 plan is `docs/dev/n6-statistics-plan.md` at SHA-256 +`15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304`. +The format authority is Apache `parquet-format` commit +`c47e2a66e88943fc46fde1b028a9432f14fdf5c0`. The checked-in +`thrift/parquet.thrift` has SHA-256 +`53bb8fc9b96469d7ca694121ead839e449e5156d7bf79f0df728cdd72796df38`. + +The harness uses Apache Thrift 0.23.0 for both generation and decoding. It uses +Eclipse Temurin 21.0.8+9 for Java compilation and execution. The download URLs +and every expected digest are in `toolchain.env`. + +| Input | Version or source | SHA-256 | +| --- | --- | --- | +| Apache Thrift source | 0.23.0 | `1859d932d2ae1f13d16c5a196931208c116310a5ff50f2bfd11d3db03be8f46f` | +| Homebrew compiler formula | core commit `bd296b14f19462baf03d5d96920209087ca99fa0` | `3929691e8a327dd0e61f41c8d7a6cccdd30ab1213b01a73d0148b195d752b209` | +| Apache Thrift Java runtime | 0.23.0 | `8b41b67a5ff13c371ab18b6d34506121dcecf11372829f7d50115cfb1bf72d42` | +| Apache Thrift runtime POM | 0.23.0 | `eeadf7b9d1e22ac01985fe552384ceecf44018cae93e7a23d6b0466f856330f3` | +| SLF4J API | 1.7.36 | `d3ef575e3e4979678dc01bf1dcce51021493b4d11fb7f1be8ad982877c16a1c0` | +| SLF4J no-op binding | 1.7.36 | `c214958b07816cb4412b30c7bdbd4308ffdc6ba2a83767b8f3a9229cbd9274d6` | +| Generated Java source manifest | exact 2.13 IDL, 67 files | `f738c7346ad1dd70faafd54815f829b8587a2b0397ff6b6e6710a3a7276cac09` | +| Eclipse Temurin JDK archive | 21.0.8+9, macOS arm64 | `59422c2292ae4e76b87e00d8808dbe49cffa39af731e08bb0292ddb0af4e0261` | +| Pinned JDK `bin/java` | 21.0.8+9, macOS arm64 | `0045ae168ee132bbf469a26fb17dac6d1dee431c9b7826474f3b6ee574a997c9` | +| Pinned JDK `bin/javac` | 21.0.8+9, macOS arm64 | `7be7937fc6bae0ca89f0866f9ce94fc40a935dfb87806d3c701eca3402cfb90a` | +| [`jdk-darwin-arm64-sequoia.manifest`](jdk-darwin-arm64-sequoia.manifest) | 542 files and directories with type, mode, path, content or link target | `d595de66a27187223eb987765fc6c9c341d509ecd15de704656a2980bc6217bc` | + +The compiler is the 0.23.0 binary from the content-addressed Homebrew bottle. +The formula source is pinned at Homebrew core commit +`bd296b14f19462baf03d5d96920209087ca99fa0`. + +| Validated compiler platform | Bottle SHA-256 | Extracted compiler SHA-256 | +| --- | --- | --- | +| macOS 15 arm64 | `dd6ed015e1b7a980c3dfa2b0dd1c01d563a8cf73bdb7f3de87d0cc1656fc1e1b` | `5ee94e75371f7d0b2467db3acdb67b8b3814fcae3748c0ef078a490a15c57e11` | + +The harness rejects other hosts. This avoids claiming support for toolchain +artifacts that have not completed this exact self-test. The initial exact run +used macOS 15.6.1 arm64. Add another platform only after its JDK archive, +compiler bottle, extracted executables, and full self-test are pinned. + +The generated sources and downloaded tools stay under the ignored `build/` +directory. `scripts/generate.sh --check` regenerates all 67 classes and checks +the canonical per-file digest manifest. The runtime classpath contains only +the scanner classes, generated 2.13 classes, `libthrift`, `slf4j-api`, and +`slf4j-nop`. + +The scripts always invoke the downloaded and verified JDK. They do not use a +host `java` or `javac`. Compilation uses the pinned `javac --release 11`. + +## Use + +The first command downloads the fixed toolchain artifacts. Later runs use the +verified local cache. + +```sh +./test/conformance/n6/oracles/raw-java/check.sh +./test/conformance/n6/oracles/raw-java/run.sh scan \ + --input path/to/file-or-directory \ + --output raw-footer-evidence.jsonl +``` + +For a directory input, the scanner selects regular files whose names end in +`.parquet`. It sorts relative path labels with Java string order. It rejects a +duplicate label across inputs. If `--output` is absent, it writes JSONL to +standard output. It collects all evidence before output and replaces an output +file only after every input succeeds. + +The output must not be the same path, hard link, or symbolic-link target as an +input. Output replacement requires an atomic move. If the file system does not +support it, the scan fails and leaves the old output unchanged. + +`check.sh` performs these checks: + +- exact plan, IDL, compiler, runtime, and generated-source hashes; +- exact JDK archive, executable, release-file, and full extracted-tree hashes, + plus rejection and recovery from fake cached Java and Thrift executables and + a corrupted `lib/modules` image; +- a clean Java 11 compilation; +- two byte-identical scans of the generated fixture; +- equality with `expected/self-test.jsonl`; +- direct field-ID, byte, bit-pattern, and presence assertions; +- raw ColumnOrder states for IDs 1 and 2, an unknown ID, a wrong wire type, and + an empty union; +- rejection of direct, hard-link, and symbolic-link output aliases without an + input-byte change; +- stable two-pass input hashing and deterministic mutation-race rejection; +- required metadata, nonnegative row-group and column counts, schema topology, + child-count, leaf/column type agreement, and duplicate-path validation; +- pre-decode Compact-Thrift depth, declared binary length, container length, + aggregate element, and aggregate binary-byte limits; +- incremental JSON character-limit enforcement before `StringBuilder` growth; +- rejection of invalid magic, invalid footer containment, trailing + Compact-Thrift bytes, and a short file; +- preservation of an existing output file after a failed scan. + +## Evidence + +`evidence.schema.json` version 3 applies to each JSONL line. Output has no timestamps or +absolute input paths. Strings use deterministic ASCII JSON escapes. SHA-256 +identifies the complete file and the exact encoded footer. + +The evidence records: + +- an ordinal and path-keyed schema leaf table. Each leaf retains its physical + type; presence and value for `type_length`, legacy `converted_type`, `scale`, + `precision`, repetition type, and field ID; and the raw LogicalType union + member with INTEGER, DECIMAL, TIME, TIMESTAMP, VARIANT, GEOMETRY, and + GEOGRAPHY parameters. A present LogicalType that the pinned generated union + cannot identify retains `present: true` with a null member and parameters; +- a schema-leaf ordinal on every matched row-group column and ColumnOrder + entry. The scanner rejects a column path or physical type that disagrees + with the leaf table; +- the exact raw `FileMetaData.column_orders` union header bytes, field ID, wire + type, state, and known member name for each entry. Any signed i16 field ID is + retained. States distinguish known, unknown, wrong-type, and empty unions; +- every `Statistics` field from ID 1 through ID 9, with separate presence and + value states; +- modern bounds at field IDs 5 and 6; +- exactness flags at field IDs 7 and 8, including present `false` versus + absent; +- null, distinct, and NaN counts, including present zero versus absent; +- exact bound bytes in PLAIN wire order, including length-prefix-free + BYTE_ARRAY values; +- Float32, Float64, and FLOAT16 bit patterns in conventional most-significant + byte first hexadecimal form, with an explicit width-valid flag. + +The self-test fixture is a metadata-only Parquet envelope. It is not a value +interoperability fixture. It includes TYPE_ORDER and IEEE_754_TOTAL_ORDER, +modern and deprecated bounds, positive and negative zero, two NaN payloads, +FLOAT16, arbitrary byte bounds, present and absent counts, all exactness +states, and INTEGER, DECIMAL, TIME, and TIMESTAMP schema parameters with legacy +annotations. + +## Boundary + +The scanner reads ordinary `PAR1` footer envelopes. It does not read encrypted +`PARE` footers. It accepts at most 10,000 input files, an 8 GiB file, a 64 MiB +footer, 1,000,000 schema or column-order entries, 1,000,000 elements in one +Compact-Thrift container, 4,000,000 aggregate container values, 64 MiB of +aggregate Compact-Thrift binary payload, 128 nested Compact-Thrift structs or +containers, and 256 MiB of retained JSON text. It hashes each open input twice +and rejects any content, size, path +identity, or modification-time change. It validates required metadata and +schema topology before it reports evidence. + +This is a local test-corpus tool. The structural preflight runs before generated +decoding and prevents allocations from oversized wire declarations. Its inputs +must still be trusted fixtures. These limits do not make it a network service +or a general hostile-input sandbox. The scanner exposes raw metadata; it does +not apply Parquet order, trust, or pruning semantics. + +This is local N6 test evidence. This directory does not publish an oracle +image and does not create or authorize a repository `oracles.lock`. diff --git a/test/conformance/n6/oracles/raw-java/check.sh b/test/conformance/n6/oracles/raw-java/check.sh new file mode 100755 index 0000000..8b2a690 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/check.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=scripts/common.sh +source "$root/scripts/common.sh" + +/bin/bash "$root/scripts/build.sh" +classpath=$(raw_java_classpath) +java_bin=$(raw_java_java) +case "$classpath" in + *parquet-avro*|*parquet-format-2.12*|*parquet-hadoop*) + echo "raw-java: parquet-java leaked into the scanner classpath" >&2 + exit 1 + ;; +esac +fixture="$RAW_JAVA_BUILD_DIR/self-test.parquet" +actual="$RAW_JAVA_BUILD_DIR/self-test.actual.jsonl" +second="$RAW_JAVA_BUILD_DIR/self-test.second.jsonl" +compiler="$RAW_JAVA_BUILD_DIR/compiler/thrift" +java_backup="$RAW_JAVA_BUILD_DIR/java.backup" +compiler_backup="$RAW_JAVA_BUILD_DIR/thrift.backup" +modules="$RAW_JAVA_JDK_DIR/lib/modules" +modules_digest=$(raw_java_sha256 "$modules") +cp "$RAW_JAVA_JDK_DIR/bin/java" "$java_backup" +cp "$compiler" "$compiler_backup" +restore_toolchain() { + [[ ! -f $java_backup ]] || mv "$java_backup" "$RAW_JAVA_JDK_DIR/bin/java" + [[ ! -f $compiler_backup ]] || mv "$compiler_backup" "$compiler" + if [[ ! -f $modules ]] || [[ $(raw_java_sha256 "$modules") != "$modules_digest" ]]; then + /bin/bash "$root/scripts/fetch-toolchain.sh" >/dev/null || true + fi +} +trap restore_toolchain EXIT +printf '#!/usr/bin/env bash\necho "Thrift version %s"\n' \ + "$RAW_JAVA_THRIFT_VERSION" > "$compiler" +chmod 0755 "$compiler" +printf '#!/usr/bin/env bash\necho "fake cached java"\n' > "$RAW_JAVA_JDK_DIR/bin/java" +chmod 0755 "$RAW_JAVA_JDK_DIR/bin/java" +/bin/bash "$root/scripts/fetch-toolchain.sh" >/dev/null +raw_java_verify "$RAW_JAVA_COMPILER_DARWIN_ARM64_SEQUOIA_SHA256" "$compiler" +raw_java_verify "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_JAVA_SHA256" \ + "$RAW_JAVA_JDK_DIR/bin/java" +rm "$java_backup" "$compiler_backup" +printf 'corrupt cached module image\n' >> "$modules" +/bin/bash "$root/scripts/fetch-toolchain.sh" >/dev/null +raw_java_verify "$modules_digest" "$modules" +raw_java_verify_tree "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_TREE_SHA256" \ + "$RAW_JAVA_JDK_DIR" +trap - EXIT +fake_cache="$RAW_JAVA_BUILD_DIR/fake-cache.jar" +printf 'fake cached artifact\n' > "$fake_cache" +if raw_java_download "$RAW_JAVA_LIBTHRIFT_URL" "$RAW_JAVA_LIBTHRIFT_SHA256" \ + "$fake_cache" >/dev/null 2>&1; then + echo "raw-java: a fake cached artifact passed hash verification" >&2 + exit 1 +fi +rm "$fake_cache" + +"$java_bin" -Dfile.encoding=UTF-8 -cp "$classpath" \ + org.julialang.parquet.n6.raw.SelfTestFixture "$fixture" +"$java_bin" -Dfile.encoding=UTF-8 -cp "$classpath" \ + org.julialang.parquet.n6.raw.RawFooterScanner \ + scan --input "$fixture" --output "$actual" +"$java_bin" -Dfile.encoding=UTF-8 -cp "$classpath" \ + org.julialang.parquet.n6.raw.RawFooterScanner \ + scan --input "$fixture" --output "$second" +cmp "$actual" "$second" +cmp "$RAW_JAVA_ROOT/expected/self-test.jsonl" "$actual" +"$java_bin" -Dfile.encoding=UTF-8 -cp "$classpath" \ + org.julialang.parquet.n6.raw.SelfTestMain "$fixture" +printf 'raw-java self-test passed.\n' diff --git a/test/conformance/n6/oracles/raw-java/evidence.schema.json b/test/conformance/n6/oracles/raw-java/evidence.schema.json new file mode 100644 index 0000000..45b3f98 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/evidence.schema.json @@ -0,0 +1,948 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://julialang.org/parquet/n6/raw-footer-evidence.schema.json", + "title": "Parquet 2.13 raw footer evidence line", + "description": "Schema for one JSONL record emitted by the isolated raw Java scanner.", + "type": "object", + "additionalProperties": false, + "required": [ + "evidence_version", + "format_commit", + "thrift_version", + "file", + "file_size", + "file_sha256", + "footer_length", + "footer_sha256", + "file_metadata" + ], + "properties": { + "evidence_version": { + "const": "parquet-2.13-raw-footer-v3" + }, + "format_commit": { + "const": "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" + }, + "thrift_version": { + "const": "0.23.0" + }, + "file": { + "type": "string", + "minLength": 1 + }, + "file_size": { + "type": "integer", + "minimum": 12 + }, + "file_sha256": { + "$ref": "#/$defs/sha256" + }, + "footer_length": { + "type": "integer", + "minimum": 0, + "maximum": 2147483647 + }, + "footer_sha256": { + "$ref": "#/$defs/sha256" + }, + "file_metadata": { + "$ref": "#/$defs/fileMetadata" + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "nullableString": { + "type": ["string", "null"] + }, + "path": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "nullablePath": { + "oneOf": [ + { + "$ref": "#/$defs/path" + }, + { + "type": "null" + } + ] + }, + "physicalType": { + "enum": [ + "BOOLEAN", + "INT32", + "INT64", + "INT96", + "FLOAT", + "DOUBLE", + "BYTE_ARRAY", + "FIXED_LEN_BYTE_ARRAY", + null + ] + }, + "logicalType": { + "enum": [ + "STRING", + "MAP", + "LIST", + "ENUM", + "DECIMAL", + "DATE", + "TIME", + "TIMESTAMP", + "INTEGER", + "UNKNOWN", + "JSON", + "BSON", + "UUID", + "FLOAT16", + "VARIANT", + "GEOMETRY", + "GEOGRAPHY", + null + ] + }, + "logicalMember": { + "enum": [ + "STRING", + "MAP", + "LIST", + "ENUM", + "DECIMAL", + "DATE", + "TIME", + "TIMESTAMP", + "INTEGER", + "UNKNOWN", + "JSON", + "BSON", + "UUID", + "FLOAT16", + "VARIANT", + "GEOMETRY", + "GEOGRAPHY" + ] + }, + "presenceInteger": { + "type": "object", + "additionalProperties": false, + "required": ["present", "value"], + "properties": { + "present": {"type": "boolean"}, + "value": {"type": ["integer", "null"]} + }, + "allOf": [ + { + "if": {"properties": {"present": {"const": false}}}, + "then": {"properties": {"value": {"type": "null"}}} + }, + { + "if": {"properties": {"present": {"const": true}}}, + "then": {"properties": {"value": {"type": "integer"}}} + } + ] + }, + "presenceString": { + "type": "object", + "additionalProperties": false, + "required": ["present", "value"], + "properties": { + "present": {"type": "boolean"}, + "value": {"type": ["string", "null"]} + }, + "allOf": [ + { + "if": {"properties": {"present": {"const": false}}}, + "then": {"properties": {"value": {"type": "null"}}} + }, + { + "if": {"properties": {"present": {"const": true}}}, + "then": {"properties": {"value": {"type": "string"}}} + } + ] + }, + "convertedTypeValue": { + "enum": [ + "UTF8", + "MAP", + "MAP_KEY_VALUE", + "LIST", + "ENUM", + "DECIMAL", + "DATE", + "TIME_MILLIS", + "TIME_MICROS", + "TIMESTAMP_MILLIS", + "TIMESTAMP_MICROS", + "UINT_8", + "UINT_16", + "UINT_32", + "UINT_64", + "INT_8", + "INT_16", + "INT_32", + "INT_64", + "JSON", + "BSON", + "INTERVAL" + ] + }, + "schemaLeaf": { + "type": "object", + "additionalProperties": false, + "required": [ + "ordinal", + "path", + "physical_type", + "type_length", + "converted_type", + "scale", + "precision", + "repetition_type", + "field_id", + "logical_type" + ], + "properties": { + "ordinal": {"type": "integer", "minimum": 0}, + "path": {"$ref": "#/$defs/path", "minItems": 1}, + "physical_type": { + "enum": [ + "BOOLEAN", + "INT32", + "INT64", + "INT96", + "FLOAT", + "DOUBLE", + "BYTE_ARRAY", + "FIXED_LEN_BYTE_ARRAY" + ] + }, + "type_length": {"$ref": "#/$defs/presenceInteger"}, + "converted_type": { + "allOf": [ + {"$ref": "#/$defs/presenceString"}, + {"properties": {"value": { + "oneOf": [ + {"$ref": "#/$defs/convertedTypeValue"}, + {"type": "null"} + ] + }}} + ] + }, + "scale": {"$ref": "#/$defs/presenceInteger"}, + "precision": {"$ref": "#/$defs/presenceInteger"}, + "repetition_type": { + "allOf": [ + {"$ref": "#/$defs/presenceString"}, + {"properties": {"value": { + "enum": ["REQUIRED", "OPTIONAL", "REPEATED", null] + }}} + ] + }, + "field_id": {"$ref": "#/$defs/presenceInteger"}, + "logical_type": {"$ref": "#/$defs/leafLogicalType"} + } + }, + "leafLogicalType": { + "type": "object", + "additionalProperties": false, + "required": ["present", "member", "parameters"], + "properties": { + "present": {"type": "boolean"}, + "member": { + "oneOf": [ + {"$ref": "#/$defs/logicalMember"}, + {"type": "null"} + ] + }, + "parameters": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/integerParameters"}, + {"$ref": "#/$defs/decimalParameters"}, + {"$ref": "#/$defs/timeParameters"}, + {"$ref": "#/$defs/variantParameters"}, + {"$ref": "#/$defs/geometryParameters"}, + {"$ref": "#/$defs/geographyParameters"} + ] + } + }, + "allOf": [ + { + "if": {"properties": {"present": {"const": false}}}, + "then": {"properties": {"member": {"type": "null"}, "parameters": {"type": "null"}}} + }, + { + "if": {"properties": {"present": {"const": true}}}, + "then": {"properties": {"member": { + "oneOf": [ + {"$ref": "#/$defs/logicalMember"}, + {"type": "null"} + ] + }}} + }, + { + "if": {"properties": {"member": {"type": "null"}}}, + "then": {"properties": {"parameters": {"type": "null"}}} + }, + { + "if": {"properties": {"member": {"enum": ["STRING", "MAP", "LIST", "ENUM", "DATE", "UNKNOWN", "JSON", "BSON", "UUID", "FLOAT16"]}}}, + "then": {"properties": {"parameters": {"type": "null"}}} + }, + { + "if": {"properties": {"member": {"const": "INTEGER"}}}, + "then": {"properties": {"parameters": {"$ref": "#/$defs/integerParameters"}}} + }, + { + "if": {"properties": {"member": {"const": "DECIMAL"}}}, + "then": {"properties": {"parameters": {"$ref": "#/$defs/decimalParameters"}}} + }, + { + "if": {"properties": {"member": {"enum": ["TIME", "TIMESTAMP"]}}}, + "then": {"properties": {"parameters": {"$ref": "#/$defs/timeParameters"}}} + }, + { + "if": {"properties": {"member": {"const": "VARIANT"}}}, + "then": {"properties": {"parameters": {"$ref": "#/$defs/variantParameters"}}} + }, + { + "if": {"properties": {"member": {"const": "GEOMETRY"}}}, + "then": {"properties": {"parameters": {"$ref": "#/$defs/geometryParameters"}}} + }, + { + "if": {"properties": {"member": {"const": "GEOGRAPHY"}}}, + "then": {"properties": {"parameters": {"$ref": "#/$defs/geographyParameters"}}} + } + ] + }, + "integerParameters": { + "type": "object", + "additionalProperties": false, + "required": ["bit_width", "is_signed"], + "properties": { + "bit_width": {"type": "integer", "minimum": -128, "maximum": 127}, + "is_signed": {"type": "boolean"} + } + }, + "decimalParameters": { + "type": "object", + "additionalProperties": false, + "required": ["scale", "precision"], + "properties": { + "scale": {"type": "integer"}, + "precision": {"type": "integer"} + } + }, + "timeParameters": { + "type": "object", + "additionalProperties": false, + "required": ["unit", "is_adjusted_to_utc"], + "properties": { + "unit": {"enum": ["MILLIS", "MICROS", "NANOS"]}, + "is_adjusted_to_utc": {"type": "boolean"} + } + }, + "variantParameters": { + "type": "object", + "additionalProperties": false, + "required": ["specification_version"], + "properties": { + "specification_version": {"$ref": "#/$defs/presenceInteger"} + } + }, + "geometryParameters": { + "type": "object", + "additionalProperties": false, + "required": ["crs"], + "properties": { + "crs": {"$ref": "#/$defs/presenceString"} + } + }, + "geographyParameters": { + "type": "object", + "additionalProperties": false, + "required": ["crs", "algorithm"], + "properties": { + "crs": {"$ref": "#/$defs/presenceString"}, + "algorithm": { + "allOf": [ + {"$ref": "#/$defs/presenceString"}, + {"properties": {"value": { + "enum": ["SPHERICAL", "VINCENTY", "THOMAS", "ANDOYER", "KARNEY", null] + }}} + ] + } + } + }, + "fileMetadata": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "num_rows", + "created_by", + "schema_leaf_count", + "schema_leaves", + "column_orders", + "row_group_count", + "row_groups" + ], + "properties": { + "version": { + "type": "integer", + "minimum": 1 + }, + "num_rows": { + "type": "integer", + "minimum": 0 + }, + "created_by": { + "type": "object", + "additionalProperties": false, + "required": ["present", "value"], + "properties": { + "present": { + "type": "boolean" + }, + "value": { + "$ref": "#/$defs/nullableString" + } + }, + "allOf": [ + { + "if": {"properties": {"present": {"const": false}}}, + "then": {"properties": {"value": {"type": "null"}}} + }, + { + "if": {"properties": {"present": {"const": true}}}, + "then": {"properties": {"value": {"type": "string"}}} + } + ] + }, + "schema_leaf_count": { + "type": "integer", + "minimum": 0 + }, + "schema_leaves": { + "type": "array", + "items": { + "$ref": "#/$defs/schemaLeaf" + } + }, + "column_orders": { + "$ref": "#/$defs/columnOrders" + }, + "row_group_count": { + "type": "integer", + "minimum": 0 + }, + "row_groups": { + "type": "array", + "items": { + "$ref": "#/$defs/rowGroup" + } + } + } + }, + "columnOrders": { + "type": "object", + "additionalProperties": false, + "required": ["present", "count", "values"], + "properties": { + "present": { + "type": "boolean" + }, + "count": { + "type": ["integer", "null"], + "minimum": 0 + }, + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/columnOrder" + } + } + }, + "allOf": [ + { + "if": {"properties": {"present": {"const": false}}}, + "then": { + "properties": { + "count": {"type": "null"}, + "values": {"maxItems": 0} + } + } + }, + { + "if": {"properties": {"present": {"const": true}}}, + "then": {"properties": {"count": {"type": "integer", "minimum": 0}}} + } + ] + }, + "columnOrder": { + "type": "object", + "additionalProperties": false, + "required": [ + "ordinal", + "schema_leaf_ordinal", + "state", + "field_id", + "wire_type", + "header_hex", + "member", + "path", + "physical_type", + "logical_type" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 0 + }, + "schema_leaf_ordinal": { + "type": ["integer", "null"], + "minimum": 0 + }, + "field_id": { + "oneOf": [ + {"type": "integer", "minimum": -32768, "maximum": 32767}, + {"type": "null"} + ] + }, + "state": { + "enum": ["known", "unknown", "wrong_type", "empty"] + }, + "wire_type": { + "oneOf": [ + {"type": "integer", "minimum": 0, "maximum": 16}, + {"type": "null"} + ] + }, + "header_hex": { + "type": "string", + "pattern": "^(?:[0-9a-f]{2})+$" + }, + "member": { + "enum": ["TYPE_ORDER", "IEEE_754_TOTAL_ORDER", null] + }, + "path": { + "$ref": "#/$defs/nullablePath" + }, + "physical_type": { + "$ref": "#/$defs/physicalType" + }, + "logical_type": { + "$ref": "#/$defs/logicalType" + } + }, + "allOf": [ + { + "if": {"properties": {"state": {"const": "known"}}}, + "then": { + "oneOf": [ + { + "properties": { + "field_id": {"const": 1}, + "wire_type": {"const": 12}, + "member": {"const": "TYPE_ORDER"} + } + }, + { + "properties": { + "field_id": {"const": 2}, + "wire_type": {"const": 12}, + "member": {"const": "IEEE_754_TOTAL_ORDER"} + } + } + ] + } + }, + { + "if": {"properties": {"state": {"const": "unknown"}}}, + "then": { + "properties": { + "field_id": {"type": "integer", "not": {"enum": [1, 2]}}, + "wire_type": {"type": "integer"}, + "member": {"type": "null"} + } + } + }, + { + "if": {"properties": {"state": {"const": "wrong_type"}}}, + "then": { + "properties": { + "field_id": {"enum": [1, 2]}, + "wire_type": {"type": "integer", "not": {"const": 12}}, + "member": {"type": "null"} + } + } + }, + { + "if": {"properties": {"state": {"const": "empty"}}}, + "then": { + "properties": { + "field_id": {"type": "null"}, + "wire_type": {"type": "null"}, + "member": {"type": "null"} + } + } + } + ] + }, + "rowGroup": { + "type": "object", + "additionalProperties": false, + "required": ["ordinal", "num_rows", "column_count", "columns"], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 0 + }, + "num_rows": { + "type": "integer", + "minimum": 0 + }, + "column_count": { + "type": "integer", + "minimum": 0 + }, + "columns": { + "type": "array", + "items": { + "$ref": "#/$defs/column" + } + } + } + }, + "column": { + "type": "object", + "additionalProperties": false, + "required": [ + "ordinal", + "schema_leaf_ordinal", + "metadata_present", + "path", + "physical_type", + "logical_type", + "num_values", + "statistics" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 0 + }, + "schema_leaf_ordinal": { + "type": ["integer", "null"], + "minimum": 0 + }, + "metadata_present": { + "type": "boolean" + }, + "path": { + "$ref": "#/$defs/nullablePath" + }, + "physical_type": { + "$ref": "#/$defs/physicalType" + }, + "logical_type": { + "$ref": "#/$defs/logicalType" + }, + "num_values": { + "type": ["integer", "null"], + "minimum": 0 + }, + "statistics": { + "$ref": "#/$defs/statistics" + } + } + }, + "statistics": { + "type": "object", + "additionalProperties": false, + "required": ["present", "fields"], + "properties": { + "present": { + "type": "boolean" + }, + "fields": { + "type": "array", + "minItems": 9, + "maxItems": 9, + "prefixItems": [ + { + "$ref": "#/$defs/maxField" + }, + { + "$ref": "#/$defs/minField" + }, + { + "$ref": "#/$defs/nullCountField" + }, + { + "$ref": "#/$defs/distinctCountField" + }, + { + "$ref": "#/$defs/maxValueField" + }, + { + "$ref": "#/$defs/minValueField" + }, + { + "$ref": "#/$defs/maxExactField" + }, + { + "$ref": "#/$defs/minExactField" + }, + { + "$ref": "#/$defs/nanCountField" + } + ], + "items": false + } + } + }, + "fieldBase": { + "type": "object", + "additionalProperties": false, + "required": ["field_id", "name", "present", "value"], + "properties": { + "field_id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "present": { + "type": "boolean" + }, + "value": {} + }, + "allOf": [ + { + "if": {"properties": {"present": {"const": false}}}, + "then": {"properties": {"value": {"type": "null"}}} + }, + { + "if": {"properties": {"present": {"const": true}}}, + "then": {"properties": {"value": {"not": {"type": "null"}}}} + } + ] + }, + "binaryFieldBase": { + "allOf": [ + { + "$ref": "#/$defs/fieldBase" + }, + { + "properties": { + "value": { + "oneOf": [ + { + "$ref": "#/$defs/binaryValue" + }, + { + "type": "null" + } + ] + } + } + } + ] + }, + "integerFieldBase": { + "allOf": [ + { + "$ref": "#/$defs/fieldBase" + }, + { + "properties": { + "value": { + "type": ["integer", "null"] + } + } + } + ] + }, + "booleanFieldBase": { + "allOf": [ + { + "$ref": "#/$defs/fieldBase" + }, + { + "properties": { + "value": { + "type": ["boolean", "null"] + } + } + } + ] + }, + "maxField": { + "allOf": [ + { + "$ref": "#/$defs/binaryFieldBase" + }, + { + "properties": { + "field_id": {"const": 1}, + "name": {"const": "max"} + } + } + ] + }, + "minField": { + "allOf": [ + { + "$ref": "#/$defs/binaryFieldBase" + }, + { + "properties": { + "field_id": {"const": 2}, + "name": {"const": "min"} + } + } + ] + }, + "nullCountField": { + "allOf": [ + { + "$ref": "#/$defs/integerFieldBase" + }, + { + "properties": { + "field_id": {"const": 3}, + "name": {"const": "null_count"} + } + } + ] + }, + "distinctCountField": { + "allOf": [ + { + "$ref": "#/$defs/integerFieldBase" + }, + { + "properties": { + "field_id": {"const": 4}, + "name": {"const": "distinct_count"} + } + } + ] + }, + "maxValueField": { + "allOf": [ + { + "$ref": "#/$defs/binaryFieldBase" + }, + { + "properties": { + "field_id": {"const": 5}, + "name": {"const": "max_value"} + } + } + ] + }, + "minValueField": { + "allOf": [ + { + "$ref": "#/$defs/binaryFieldBase" + }, + { + "properties": { + "field_id": {"const": 6}, + "name": {"const": "min_value"} + } + } + ] + }, + "maxExactField": { + "allOf": [ + { + "$ref": "#/$defs/booleanFieldBase" + }, + { + "properties": { + "field_id": {"const": 7}, + "name": {"const": "is_max_value_exact"} + } + } + ] + }, + "minExactField": { + "allOf": [ + { + "$ref": "#/$defs/booleanFieldBase" + }, + { + "properties": { + "field_id": {"const": 8}, + "name": {"const": "is_min_value_exact"} + } + } + ] + }, + "nanCountField": { + "allOf": [ + { + "$ref": "#/$defs/integerFieldBase" + }, + { + "properties": { + "field_id": {"const": 9}, + "name": {"const": "nan_count"} + } + } + ] + }, + "binaryValue": { + "type": "object", + "additionalProperties": false, + "required": ["hex", "byte_length", "float_bits"], + "properties": { + "hex": { + "type": "string", + "pattern": "^(?:[0-9a-f]{2})*$" + }, + "byte_length": { + "type": "integer", + "minimum": 0 + }, + "float_bits": { + "oneOf": [ + { + "$ref": "#/$defs/floatBits" + }, + { + "type": "null" + } + ] + } + } + }, + "floatBits": { + "type": "object", + "additionalProperties": false, + "required": ["width", "valid_width", "hex"], + "properties": { + "width": { + "enum": [16, 32, 64] + }, + "valid_width": { + "type": "boolean" + }, + "hex": { + "type": ["string", "null"], + "pattern": "^0x[0-9a-f]+$" + } + } + } + } +} diff --git a/test/conformance/n6/oracles/raw-java/expected/self-test.jsonl b/test/conformance/n6/oracles/raw-java/expected/self-test.jsonl new file mode 100644 index 0000000..6a8b38d --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/expected/self-test.jsonl @@ -0,0 +1 @@ +{"evidence_version":"parquet-2.13-raw-footer-v3","format_commit":"c47e2a66e88943fc46fde1b028a9432f14fdf5c0","thrift_version":"0.23.0","file":"self-test.parquet","file_size":614,"file_sha256":"75ede6cf360b24e01eb1de946d2671a116b14a6a74cf907414c2d7284fc437b1","footer_length":602,"footer_sha256":"9525ef48b45f6b00a6b0def6760aff970e3ebc6d54add139795464615462ff5d","file_metadata":{"version":1,"num_rows":3,"created_by":{"present":true,"value":"raw-java self-test \u03c0"},"schema_leaf_count":8,"schema_leaves":[{"ordinal":0,"path":["float32"],"physical_type":"FLOAT","type_length":{"present":false,"value":null},"converted_type":{"present":false,"value":null},"scale":{"present":false,"value":null},"precision":{"present":false,"value":null},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":false,"value":null},"logical_type":{"present":false,"member":null,"parameters":null}},{"ordinal":1,"path":["float64"],"physical_type":"DOUBLE","type_length":{"present":false,"value":null},"converted_type":{"present":false,"value":null},"scale":{"present":false,"value":null},"precision":{"present":false,"value":null},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":false,"value":null},"logical_type":{"present":false,"member":null,"parameters":null}},{"ordinal":2,"path":["float16"],"physical_type":"FIXED_LEN_BYTE_ARRAY","type_length":{"present":true,"value":2},"converted_type":{"present":false,"value":null},"scale":{"present":false,"value":null},"precision":{"present":false,"value":null},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":false,"value":null},"logical_type":{"present":true,"member":"FLOAT16","parameters":null}},{"ordinal":3,"path":["binary"],"physical_type":"BYTE_ARRAY","type_length":{"present":false,"value":null},"converted_type":{"present":false,"value":null},"scale":{"present":false,"value":null},"precision":{"present":false,"value":null},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":false,"value":null},"logical_type":{"present":false,"member":null,"parameters":null}},{"ordinal":4,"path":["uint16"],"physical_type":"INT32","type_length":{"present":true,"value":16},"converted_type":{"present":true,"value":"UINT_16"},"scale":{"present":false,"value":null},"precision":{"present":false,"value":null},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":true,"value":17},"logical_type":{"present":true,"member":"INTEGER","parameters":{"bit_width":16,"is_signed":false}}},{"ordinal":5,"path":["decimal4"],"physical_type":"FIXED_LEN_BYTE_ARRAY","type_length":{"present":true,"value":4},"converted_type":{"present":true,"value":"DECIMAL"},"scale":{"present":true,"value":2},"precision":{"present":true,"value":9},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":false,"value":null},"logical_type":{"present":true,"member":"DECIMAL","parameters":{"scale":2,"precision":9}}},{"ordinal":6,"path":["time64"],"physical_type":"INT64","type_length":{"present":false,"value":null},"converted_type":{"present":true,"value":"TIME_MICROS"},"scale":{"present":false,"value":null},"precision":{"present":false,"value":null},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":false,"value":null},"logical_type":{"present":true,"member":"TIME","parameters":{"unit":"MICROS","is_adjusted_to_utc":true}}},{"ordinal":7,"path":["timestamp64"],"physical_type":"INT64","type_length":{"present":false,"value":null},"converted_type":{"present":false,"value":null},"scale":{"present":false,"value":null},"precision":{"present":false,"value":null},"repetition_type":{"present":true,"value":"OPTIONAL"},"field_id":{"present":false,"value":null},"logical_type":{"present":true,"member":"TIMESTAMP","parameters":{"unit":"NANOS","is_adjusted_to_utc":false}}}],"column_orders":{"present":true,"count":8,"values":[{"ordinal":0,"schema_leaf_ordinal":0,"state":"known","field_id":1,"wire_type":12,"header_hex":"1c","member":"TYPE_ORDER","path":["float32"],"physical_type":"FLOAT","logical_type":null},{"ordinal":1,"schema_leaf_ordinal":1,"state":"known","field_id":2,"wire_type":12,"header_hex":"2c","member":"IEEE_754_TOTAL_ORDER","path":["float64"],"physical_type":"DOUBLE","logical_type":null},{"ordinal":2,"schema_leaf_ordinal":2,"state":"known","field_id":2,"wire_type":12,"header_hex":"2c","member":"IEEE_754_TOTAL_ORDER","path":["float16"],"physical_type":"FIXED_LEN_BYTE_ARRAY","logical_type":"FLOAT16"},{"ordinal":3,"schema_leaf_ordinal":3,"state":"known","field_id":1,"wire_type":12,"header_hex":"1c","member":"TYPE_ORDER","path":["binary"],"physical_type":"BYTE_ARRAY","logical_type":null},{"ordinal":4,"schema_leaf_ordinal":4,"state":"known","field_id":1,"wire_type":12,"header_hex":"1c","member":"TYPE_ORDER","path":["uint16"],"physical_type":"INT32","logical_type":"INTEGER"},{"ordinal":5,"schema_leaf_ordinal":5,"state":"known","field_id":1,"wire_type":12,"header_hex":"1c","member":"TYPE_ORDER","path":["decimal4"],"physical_type":"FIXED_LEN_BYTE_ARRAY","logical_type":"DECIMAL"},{"ordinal":6,"schema_leaf_ordinal":6,"state":"known","field_id":1,"wire_type":12,"header_hex":"1c","member":"TYPE_ORDER","path":["time64"],"physical_type":"INT64","logical_type":"TIME"},{"ordinal":7,"schema_leaf_ordinal":7,"state":"known","field_id":1,"wire_type":12,"header_hex":"1c","member":"TYPE_ORDER","path":["timestamp64"],"physical_type":"INT64","logical_type":"TIMESTAMP"}]},"row_group_count":1,"row_groups":[{"ordinal":0,"num_rows":3,"column_count":8,"columns":[{"ordinal":0,"schema_leaf_ordinal":0,"metadata_present":true,"path":["float32"],"physical_type":"FLOAT","logical_type":null,"num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":true,"value":{"hex":"0000803f","byte_length":4,"float_bits":{"width":32,"valid_width":true,"hex":"0x3f800000"}}},{"field_id":2,"name":"min","present":true,"value":{"hex":"000080bf","byte_length":4,"float_bits":{"width":32,"valid_width":true,"hex":"0xbf800000"}}},{"field_id":3,"name":"null_count","present":true,"value":1},{"field_id":4,"name":"distinct_count","present":true,"value":2},{"field_id":5,"name":"max_value","present":true,"value":{"hex":"00000000","byte_length":4,"float_bits":{"width":32,"valid_width":true,"hex":"0x00000000"}}},{"field_id":6,"name":"min_value","present":true,"value":{"hex":"00000080","byte_length":4,"float_bits":{"width":32,"valid_width":true,"hex":"0x80000000"}}},{"field_id":7,"name":"is_max_value_exact","present":true,"value":true},{"field_id":8,"name":"is_min_value_exact","present":true,"value":false},{"field_id":9,"name":"nan_count","present":true,"value":0}]}},{"ordinal":1,"schema_leaf_ordinal":1,"metadata_present":true,"path":["float64"],"physical_type":"DOUBLE","logical_type":null,"num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":false,"value":null},{"field_id":2,"name":"min","present":false,"value":null},{"field_id":3,"name":"null_count","present":true,"value":1},{"field_id":4,"name":"distinct_count","present":false,"value":null},{"field_id":5,"name":"max_value","present":true,"value":{"hex":"420000000000f87f","byte_length":8,"float_bits":{"width":64,"valid_width":true,"hex":"0x7ff8000000000042"}}},{"field_id":6,"name":"min_value","present":true,"value":{"hex":"010000000000f8ff","byte_length":8,"float_bits":{"width":64,"valid_width":true,"hex":"0xfff8000000000001"}}},{"field_id":7,"name":"is_max_value_exact","present":true,"value":true},{"field_id":8,"name":"is_min_value_exact","present":true,"value":true},{"field_id":9,"name":"nan_count","present":true,"value":2}]}},{"ordinal":2,"schema_leaf_ordinal":2,"metadata_present":true,"path":["float16"],"physical_type":"FIXED_LEN_BYTE_ARRAY","logical_type":"FLOAT16","num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":false,"value":null},{"field_id":2,"name":"min","present":false,"value":null},{"field_id":3,"name":"null_count","present":true,"value":0},{"field_id":4,"name":"distinct_count","present":true,"value":2},{"field_id":5,"name":"max_value","present":true,"value":{"hex":"007c","byte_length":2,"float_bits":{"width":16,"valid_width":true,"hex":"0x7c00"}}},{"field_id":6,"name":"min_value","present":true,"value":{"hex":"0080","byte_length":2,"float_bits":{"width":16,"valid_width":true,"hex":"0x8000"}}},{"field_id":7,"name":"is_max_value_exact","present":false,"value":null},{"field_id":8,"name":"is_min_value_exact","present":true,"value":true},{"field_id":9,"name":"nan_count","present":true,"value":1}]}},{"ordinal":3,"schema_leaf_ordinal":3,"metadata_present":true,"path":["binary"],"physical_type":"BYTE_ARRAY","logical_type":null,"num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":true,"value":{"hex":"ff","byte_length":1,"float_bits":null}},{"field_id":2,"name":"min","present":true,"value":{"hex":"00","byte_length":1,"float_bits":null}},{"field_id":3,"name":"null_count","present":false,"value":null},{"field_id":4,"name":"distinct_count","present":true,"value":3},{"field_id":5,"name":"max_value","present":true,"value":{"hex":"ff007f","byte_length":3,"float_bits":null}},{"field_id":6,"name":"min_value","present":true,"value":{"hex":"00ff","byte_length":2,"float_bits":null}},{"field_id":7,"name":"is_max_value_exact","present":true,"value":false},{"field_id":8,"name":"is_min_value_exact","present":false,"value":null},{"field_id":9,"name":"nan_count","present":false,"value":null}]}},{"ordinal":4,"schema_leaf_ordinal":4,"metadata_present":true,"path":["uint16"],"physical_type":"INT32","logical_type":"INTEGER","num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":false,"value":null},{"field_id":2,"name":"min","present":false,"value":null},{"field_id":3,"name":"null_count","present":true,"value":0},{"field_id":4,"name":"distinct_count","present":false,"value":null},{"field_id":5,"name":"max_value","present":false,"value":null},{"field_id":6,"name":"min_value","present":false,"value":null},{"field_id":7,"name":"is_max_value_exact","present":false,"value":null},{"field_id":8,"name":"is_min_value_exact","present":false,"value":null},{"field_id":9,"name":"nan_count","present":false,"value":null}]}},{"ordinal":5,"schema_leaf_ordinal":5,"metadata_present":true,"path":["decimal4"],"physical_type":"FIXED_LEN_BYTE_ARRAY","logical_type":"DECIMAL","num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":false,"value":null},{"field_id":2,"name":"min","present":false,"value":null},{"field_id":3,"name":"null_count","present":true,"value":0},{"field_id":4,"name":"distinct_count","present":false,"value":null},{"field_id":5,"name":"max_value","present":false,"value":null},{"field_id":6,"name":"min_value","present":false,"value":null},{"field_id":7,"name":"is_max_value_exact","present":false,"value":null},{"field_id":8,"name":"is_min_value_exact","present":false,"value":null},{"field_id":9,"name":"nan_count","present":false,"value":null}]}},{"ordinal":6,"schema_leaf_ordinal":6,"metadata_present":true,"path":["time64"],"physical_type":"INT64","logical_type":"TIME","num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":false,"value":null},{"field_id":2,"name":"min","present":false,"value":null},{"field_id":3,"name":"null_count","present":true,"value":0},{"field_id":4,"name":"distinct_count","present":false,"value":null},{"field_id":5,"name":"max_value","present":false,"value":null},{"field_id":6,"name":"min_value","present":false,"value":null},{"field_id":7,"name":"is_max_value_exact","present":false,"value":null},{"field_id":8,"name":"is_min_value_exact","present":false,"value":null},{"field_id":9,"name":"nan_count","present":false,"value":null}]}},{"ordinal":7,"schema_leaf_ordinal":7,"metadata_present":true,"path":["timestamp64"],"physical_type":"INT64","logical_type":"TIMESTAMP","num_values":3,"statistics":{"present":true,"fields":[{"field_id":1,"name":"max","present":false,"value":null},{"field_id":2,"name":"min","present":false,"value":null},{"field_id":3,"name":"null_count","present":true,"value":0},{"field_id":4,"name":"distinct_count","present":false,"value":null},{"field_id":5,"name":"max_value","present":false,"value":null},{"field_id":6,"name":"min_value","present":false,"value":null},{"field_id":7,"name":"is_max_value_exact","present":false,"value":null},{"field_id":8,"name":"is_min_value_exact","present":false,"value":null},{"field_id":9,"name":"nan_count","present":false,"value":null}]}}]}]}} diff --git a/test/conformance/n6/oracles/raw-java/jdk-darwin-arm64-sequoia.manifest b/test/conformance/n6/oracles/raw-java/jdk-darwin-arm64-sequoia.manifest new file mode 100644 index 0000000..233a346 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/jdk-darwin-arm64-sequoia.manifest @@ -0,0 +1,542 @@ +file 644 NOTICE c02756bcd9fa8191bf0fda4451bc018414dd44ee35bf09922c24377a475e4b5a +directory 755 bin +file 755 bin/jar b4b69691321fb426e95a21bae51724e9f98ab6b67370eba953ee40c4f7512cbf +file 755 bin/jarsigner 1d4cf5b6f32b81acabd29a70bd63161bd4606a9cfbd39fb9db4ec99a49d09a04 +file 755 bin/java 0045ae168ee132bbf469a26fb17dac6d1dee431c9b7826474f3b6ee574a997c9 +file 755 bin/javac 7be7937fc6bae0ca89f0866f9ce94fc40a935dfb87806d3c701eca3402cfb90a +file 755 bin/javadoc ee2290672912850248159d785ca3c9347ce52925892edadc7acd65d0969b9794 +file 755 bin/javap 6f737b142a8ba935ed70784910336a2b420cdde20b590367024473e0097a977f +file 755 bin/jcmd 6d3832d5b7981b3e3d7d0025020e76c12556a5d959fbf6e1f48d44581a6463ca +file 755 bin/jconsole 121360cdf1eb926ccc58cbb10332625a775a88bc0d18f5eeef08fa04792e455b +file 755 bin/jdb 14837e3af780b842c8ae76e49e2bc56508f466ce23a336ca9267c2e0ec865404 +file 755 bin/jdeprscan 549c81981ef18dd705847fa3524748b50f7aa2252b00c7402b5cce0eb91843b0 +file 755 bin/jdeps a4166765b634b02494c49a5162e68b4ba7cbce3289d7dfec1f21dcbe9c709c8b +file 755 bin/jfr c0ea4611ba5dcff8f13fb6421d5a2fc60ed829cdb80a15c1af722f744f660bda +file 755 bin/jhsdb ad308af5953e951b9e67357b9f533872948a96c86335713a0a3390a5283657ac +file 755 bin/jimage d6bfbd85fd4791938456398b215ca6194c28f31d9b627d14a4b43e314b38e952 +file 755 bin/jinfo c93576648d9631e369c5e8517ea50ad0035147924a4cc990ae16d15886334153 +file 755 bin/jlink e340e853b2e9c1dc68d2a9c32e22db687101383d87ce914745fdfadfd5751097 +file 755 bin/jmap 1a4b6497735ab0d33ac77e4c7737b04be2519a5a23871c3bf42862d3bae44f64 +file 755 bin/jmod 4df608baeccbcf47530a1fee298dd721d7e23299ad1dc79d75e52f88a6548182 +file 755 bin/jpackage a0fb139c6a355b27ea8aeba61a7b602d6dcd30d93fe35a79f73af501a758ca6d +file 755 bin/jps 9053974df5e20050b8744cb7fb684f135badb1b4e5e439d79296919e814dd833 +file 755 bin/jrunscript ba88712e612a5b9b894e4f5ba5dc3d74689f1acc0eb5c2051a5aebb987073fc6 +file 755 bin/jshell c198f5f08b473cbb9c1f4567b7780691813480cb9f7bffac36df0cc5902278f7 +file 755 bin/jstack 604e8cb2e4658060d215d6f1318a8573f59fb36f98161e9d17fe3e175cb2d727 +file 755 bin/jstat 3085b64ea30def5b33590312152b749ede10ea94f1486297e9362eccf42130fe +file 755 bin/jstatd 1592705d2ab7f1c4bac6cbe67d8fad3239ddd14dd97b70d3ec8008c7816d962d +file 755 bin/jwebserver 279ee8c89523a992f660aa4f6936fbf93d6405d1b5a50b408a4da6bb49d93b48 +file 755 bin/keytool 219ccf106c030b069551405498cebd5d2db7af958a927d47d1410c363dc83e72 +file 755 bin/rmiregistry 2e8a9feae06a26ca27acb0df76b2a1066a918ed69da28e05cbdf31360454ec25 +file 755 bin/serialver 51cf5fcfb94ae3a86f91e59977a2f04aab4c34018ded75f8f03565c4df8ffc8d +directory 755 conf +file 644 conf/jaxp.properties 7d95c49465d0c836d608d02856d3b097934dfb5dc4bd2279affaf337d045b708 +file 644 conf/logging.properties b62d2733ab99556b108a1951d894c5a8d76b1ac7a00c02c388f9eb9be046c56f +directory 755 conf/management +file 644 conf/management/jmxremote.access 0c25d26ee212ca1e8c33f67c3c460d43fe849c3a1d23dbe341148517602b280c +file 644 conf/management/jmxremote.password.template 0273b6a6b9e20e6ce54c5aee70164028e0395063b2b7d39060a40b6495543dbf +file 644 conf/management/management.properties 07dffdd85b01c19bf46ca320a699aba48dd6b01043eb0bd6a9528c7993312bad +file 644 conf/net.properties 2e070ed1d97052f0ce8771ed2ef74a38d7c260a45fde7a3682c8844e5fdf58a4 +directory 755 conf/security +file 644 conf/security/java.policy d51bcab7ed301caeff3779a5e777e6019864cecee5e2abc102ef991b0de77af2 +file 644 conf/security/java.security 7518d28866628174e9e8f87a429caf7457a4423f704816ce8f340d1feedf0f34 +directory 755 conf/security/policy +file 644 conf/security/policy/README.txt 6da0747334b0fea7592fd92614b2bbc8b126535e129b1fee483774d914e98eb5 +directory 755 conf/security/policy/limited +file 644 conf/security/policy/limited/default_US_export.policy 758b930a526fc670ab7537f8c26321527050a31f5f42149a2dda623c56a0a1a9 +file 644 conf/security/policy/limited/default_local.policy 2b2627548e61316150d47ffc3e6cad465ca05b3cccd4785eb7d21aa7baa0f441 +file 644 conf/security/policy/limited/exempt_local.policy 8c3d7648abcd95a272ce12db870082937f4d7f6878d730d83cb7fbb31eb8b2c9 +directory 755 conf/security/policy/unlimited +file 644 conf/security/policy/unlimited/default_US_export.policy 758b930a526fc670ab7537f8c26321527050a31f5f42149a2dda623c56a0a1a9 +file 644 conf/security/policy/unlimited/default_local.policy 8d8a318e6d90dfd7e26612d2b6385aa704f686ca6134c551f8928418d92b851a +file 644 conf/sound.properties 299c2360b6155eb28990ec49cd21753f97e43442fe8fab03e04f3e213df43a66 +directory 755 include +file 644 include/classfile_constants.h 7056263197c68eb8493f53d88020aa550985e64816afd776e509d6bf8cf0b4a0 +directory 755 include/darwin +file 644 include/darwin/jawt_md.h 4af10927ebcfdfdfbf218bbf90c397856d251a312c1fe8f4469b056235da4dfb +file 644 include/darwin/jni_md.h 88cb5c33e306900dd35a78d5a439087123b8e91b0986bb5acb42cc9bd2fcc42e +file 644 include/jawt.h 85101d07928a589accfef9d7c261850faa8d5afc8ef262af7ec9734008f6f2f5 +file 644 include/jdwpTransport.h c42256f8596cbf3b59aa25be54bbf2ece47a5dee75d4e6153791dc5620cff2c8 +file 644 include/jni.h 99e64ebbe749e6df284f852f11b3c73f6ea97baf15120428f40f887fe0616e61 +file 644 include/jvmti.h 78dcca2689f42937c3b5e54eaf7ed2e4674cd808d1d6fd5fbd10f83829e577e8 +file 644 include/jvmticmlr.h 6ee3e52d24bdb4f4d0312dfbab3d47bd524cbdac5540a4d790cac0620c59b3c8 +directory 755 jmods +file 644 jmods/java.base.jmod a9cec5eaad6cf2af76a3ec0757719f9b033c1c2b6a1b2a5b3b0ee3e48645c1e7 +file 644 jmods/java.compiler.jmod 528548aab61bb1a606503d9cbcdaecfa3ba57cda5177e78ab31e02a5e78c5717 +file 644 jmods/java.datatransfer.jmod 8ed6393687a32f6d526d60cec505d9f0698f412408c14f4a855ec51a2bb17c73 +file 644 jmods/java.desktop.jmod 4a4eed97c7d3e40e0e51b63b42bab5af09bbe80e991e7758cda911166bd4047b +file 644 jmods/java.instrument.jmod 56927e0b4ba8b86744c26f9ed1583566acf6b300af307fce1f5c65f576ea4de8 +file 644 jmods/java.logging.jmod 11108ed5aa101206cc9f19dd10777f50acb4b99f0b5b0e71055f73a4792c0443 +file 644 jmods/java.management.jmod 8d8cc3a77c349a5fbfe4d9013fc9e71113f78337e0b42697fca7f4bed8b92710 +file 644 jmods/java.management.rmi.jmod b1119a376a45a689abe07f99d3233fbee28557803b0e7ccdd618c8feb93f306f +file 644 jmods/java.naming.jmod 4a67f122317a2f95e53b004d2bb6df3c9c4b16a609c2cc936d18ea698678862f +file 644 jmods/java.net.http.jmod 501f9791610309aa42cce7f47be6abb4841124c0d8ac1ecbaed36f309c361809 +file 644 jmods/java.prefs.jmod be746071786e55a839fa1e6f0e7577aedd76500288ee8dff0fdaefad75eed2e1 +file 644 jmods/java.rmi.jmod 4899a4aeebd42ef62559f324bfaa418fd63f05c650a1497fa5bdb6499177fa28 +file 644 jmods/java.scripting.jmod 1f802cc33b9d3e6b787b0059ff014d5b0d9debc11ae6b30fe4efe3d9fe4179e0 +file 644 jmods/java.se.jmod ca9ae6121175c35625d1d5d880e806da300234c264f53a020a5349d3ba6c915f +file 644 jmods/java.security.jgss.jmod cde5de23a249f9b52b57bfab5d12106687951ea15bcf41c8c4e1344fb9957b8a +file 644 jmods/java.security.sasl.jmod 23d2b8d0742943ec1c6f2b63b1f797385d2f33a6e4da31ba953c31787e1d0835 +file 644 jmods/java.smartcardio.jmod 004dc76ec1775b6bf0de523c9faebb2f8d653f7029e78a039f71bd6d3d4b8eab +file 644 jmods/java.sql.jmod bb044edbda73bba35f2cd5c5c482652322f9c07cf353a2ae0ecc5ba79a791ec8 +file 644 jmods/java.sql.rowset.jmod 32e0880061602f95fe13000b3b96c981f04e062472b62b712fff412ebf2be1bd +file 644 jmods/java.transaction.xa.jmod 0e1ac8446c9e2eb6d6c40c5760f1faeec70278b07c208d8ea622d02cfcab96e4 +file 644 jmods/java.xml.crypto.jmod 18e782eacb08b302df6d3c1477dd99df67af9a0920b6d320267f4b82bc8879c6 +file 644 jmods/java.xml.jmod 87c8d3c7d96f76fe1ed8277515e82f3304f810cbdcdb1a8072e5e495ab584a43 +file 644 jmods/jdk.accessibility.jmod 19368c26ce17b9f81d76fda234ee75d6a5d2c4804b69c96baeddb6723c86e48f +file 644 jmods/jdk.attach.jmod b6a85ca5cf7faa876901bd61f6560ccccde1e48b6403e25348c297f66c3c39db +file 644 jmods/jdk.charsets.jmod d8d67ba80cc27f7574fac496ee0343a3b3fc7d2ef667e3c3ee5dad68d8bd8266 +file 644 jmods/jdk.compiler.jmod e08db09f99405eec7079e6c3f46d6d3d1c493826106142ce6571968e18818af2 +file 644 jmods/jdk.crypto.cryptoki.jmod 09f15b3318d7bd6c27d2fbed4b89f23f678f2fde469b9401d9a078ecaa36c8cc +file 644 jmods/jdk.crypto.ec.jmod ef549dc74871f63c3816cc32c4a7348977529c627b728e94c6c606acba2d623d +file 644 jmods/jdk.dynalink.jmod c3545ab4b092ec77cd1bcfa7130a2f14648ae54fc8b6a1b112df4b3d4f0bb743 +file 644 jmods/jdk.editpad.jmod 095db78123cb634b67955b40bbfba89e9198241297387372e6d8f0c77b7049a9 +file 644 jmods/jdk.hotspot.agent.jmod 04d35953c1e3dd34b7a4ba9d9e3611ac089022eac3bd8aa902ec0030331b8261 +file 644 jmods/jdk.httpserver.jmod 7a1ffaddad9b9e4b7e6e366576a9b34e4481095996a41f182a6ccd897baae75a +file 644 jmods/jdk.incubator.vector.jmod f6dabde69489acf4e9285d50f37414ba9bd1cf047d7ddea12e3b5e22c803e53c +file 644 jmods/jdk.internal.ed.jmod 3343fe3377a3674615e4cbee2955a344d2ba7d9601d5bd612d0de0b34fc494cc +file 644 jmods/jdk.internal.jvmstat.jmod 06ff33c059289ff198998aac6d84dc726711b6c7efd935c17e7f32e2b00b5723 +file 644 jmods/jdk.internal.le.jmod d838ae10fb5d52261cf678052e1e86ca4f4916475a549247a00db738ec89ddae +file 644 jmods/jdk.internal.opt.jmod 8751c2b4dc6650e9a005a179ff270e18944fab5c474dd25d6165f195a8e579cb +file 644 jmods/jdk.internal.vm.ci.jmod 347cb779e3ae85b4992af88671eaf32feb0af91d81433ff62adb4bf254746af9 +file 644 jmods/jdk.internal.vm.compiler.jmod deb9258568b7a1f582cea9fbecd61fb4a1197a6a3cbc23ba242d6ba121b28412 +file 644 jmods/jdk.internal.vm.compiler.management.jmod 418bf9c6d57c87ad90a36865bf677c8db7120b10f7d6d2b793c92f8bf1a224eb +file 644 jmods/jdk.jartool.jmod 44426fa294aa260a23d6bcda8e4489d7f0f90813d63d48cb21ce4b9918261e67 +file 644 jmods/jdk.javadoc.jmod bd9a9b56417245741947c2e56359e437aee2453e3f16c7951f66d1b7f73524b0 +file 644 jmods/jdk.jcmd.jmod 2db5fd058f90c4a717436e6cfcdfaf77385fed6cf1712a3d398201cee7169bc4 +file 644 jmods/jdk.jconsole.jmod 53636731b69eb0c74bcb2bec39ab441b1a4c7e6c8117e935c010a1468906be45 +file 644 jmods/jdk.jdeps.jmod 5ca9c57ec4ac50d5fc2d1ce7fd577378a4e37f4cd57d05c5bb4087401d5545d2 +file 644 jmods/jdk.jdi.jmod b09706c9a098c7b7b46aa1382bf4cf9c6779f02f9de610eb7741c2ac7cd8cf26 +file 644 jmods/jdk.jdwp.agent.jmod ff3caa87015d98e4069c08c4f3d217410f8b5160e54ddc2785b4dd11514193d3 +file 644 jmods/jdk.jfr.jmod 1a17090c5ecda1f5da0c2065673c2cf86e3936bc4c4303b95a98b6d694ea9bc3 +file 644 jmods/jdk.jlink.jmod 04ea7b463b591ff649ab37fc4608ca8c81f9998ab47b006e224f097cbc1067e4 +file 644 jmods/jdk.jpackage.jmod 2bc23bf5c176f38320c3c7e24df67236185550bd28a65f0cdb07c1d33a959fdc +file 644 jmods/jdk.jshell.jmod 5c47a748016a70d5dce88219cc5d59f29866364e5f6e245601f4c4ea53cb47d1 +file 644 jmods/jdk.jsobject.jmod d128cf6bd46f0648231ae1c0d42f2c2b13264450419ff1b247214c1227b8b3db +file 644 jmods/jdk.jstatd.jmod 47529ccd35c77f43df03cbbe814c89ad85abe37554c12a677af2ad67a880cf88 +file 644 jmods/jdk.localedata.jmod 710e7b7951f5e203d191af837cd4e73d7c9d8602fa7c39c7711c70027146a14b +file 644 jmods/jdk.management.agent.jmod 430778dbb65b8752a0875b15bccb47a37f4d23f89eaa1625d2ce6072eefe1e20 +file 644 jmods/jdk.management.jfr.jmod 9bfa631916d1af06207f819e743a5132d872993d602da6af2b0e4d884d40a49f +file 644 jmods/jdk.management.jmod d0bd8dab165094eb4059dc769543ccd1518f868842debea9eaa08edfb35f1bb4 +file 644 jmods/jdk.naming.dns.jmod 1708d60f5c37258ca085b64d908677b7c20aecb9e670bff7663511000a6e529b +file 644 jmods/jdk.naming.rmi.jmod 235a9c48d526a545ac091635a6bea8c1f887d1af6685a9d582e7fd263d17e6ac +file 644 jmods/jdk.net.jmod 4c4588a5207e3bc21e4611ed3cff9f8d4396f1c6553ce7a7a60562f69ecd2056 +file 644 jmods/jdk.nio.mapmode.jmod 88ba59a6819b8fe6ced1fdd44ba646b20f6f0dbcfbe4f208463b6505673448ee +file 644 jmods/jdk.random.jmod ddb7a06ebd36d34facbb9d621e66896c2062954e91dc532966413b0312831d48 +file 644 jmods/jdk.sctp.jmod fc2d347521895f8d34114b06fe5f60006e409df30b21055478b5de587dad7b3b +file 644 jmods/jdk.security.auth.jmod c6c4a4bcc453ed1d23e38fd405a9cebddcd7d375b85e290875ff1f4516cdd634 +file 644 jmods/jdk.security.jgss.jmod 48ce934c16efdc686d323d0db2ca5588004b2498ebb0ee6d02b6cb9fe417fe72 +file 644 jmods/jdk.unsupported.desktop.jmod 728c8f7a605b83a381b2ae53898f21326808e1a1e9134be327614a25e43b63d9 +file 644 jmods/jdk.unsupported.jmod e563035a7f26606805e07f76bad66deb8a2d10f7ade7298926feaace71f946d4 +file 644 jmods/jdk.xml.dom.jmod 7d62dfab13a093063a24d2d8daa91a5bd4733747a7542132a4b79d79bba1d9c7 +file 644 jmods/jdk.zipfs.jmod a9ed84d812e02b8c0123843df6e94153c8c7a809bd0facefdb914e1719523c5d +directory 755 legal +directory 755 legal/java.base +file 444 legal/java.base/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.base/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.base/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/java.base/aes.md 45c6d4da48325edfbff3dcf71c704e504c057904435ed23c6d57046d551eb69d +file 444 legal/java.base/asm.md 683be15695bd248272d60f5b7fbe5e126a935ea6bf231a624a9aa164733e1d1d +file 444 legal/java.base/c-libutl.md bef40679922d6fdfb7e4ddb223ad6722300f6054ba737bbf6188d60fcec517f9 +file 444 legal/java.base/cldr.md 19515e14a240e022640e95b61b5095127fa9690755950b4a2b0a02e783e08163 +file 444 legal/java.base/icu.md 1bf28459c6e0af9f3429f4f8becd1668d6544055f8df240277456bc4b3d8a752 +file 444 legal/java.base/public_suffix.md d7818e02ebfc4e5cd82613e003e7ba6be2e9d5949ea4aa0ba88d4d2f7ca69999 +file 444 legal/java.base/siphash.md 5a792b5a74ad2a5f3d6a7ad8b7a841116e58a772c18bc6e392320a365b222c76 +file 444 legal/java.base/unicode.md 6f72f10d166b2c2e8a395e03e734c5afc852b59aeca73ced124f6b9c96268d53 +file 444 legal/java.base/zlib.md 809b62ba648e02302f7d9ea6b6886c10d5253ac86ad528038a50c73eada5fce2 +directory 755 legal/java.compiler +file 444 legal/java.compiler/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.compiler/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.compiler/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.datatransfer +file 444 legal/java.datatransfer/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.datatransfer/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.datatransfer/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.desktop +file 444 legal/java.desktop/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.desktop/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.desktop/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/java.desktop/colorimaging.md 04d61e3e8e71dd452ebe52008af5378d9f6640d14578aeb515dc5375973b0189 +file 444 legal/java.desktop/freetype.md f49d5fb98475239073dc657e689b85d3125a2b30bf3ee137db17541878f1cbe8 +file 444 legal/java.desktop/giflib.md 6cd971730d3047ea57f6865b7bdca2509a9876ae24d5c0ed0c4e32def5f9107e +file 444 legal/java.desktop/harfbuzz.md 54923f5f4cbfa0bfa6bd0eae88856be9305528c3f231ac8f6fce9c38de2b7740 +file 444 legal/java.desktop/jpeg.md c1dfb9719a71ad9f861f8728550542d681e25c8ef40e6393606e6e2a0c1d653a +file 444 legal/java.desktop/lcms.md fc1d7f60c7cdc96b5cadafd4f3610c6a74f2d0afa08852036929c7a1474ebab6 +file 444 legal/java.desktop/libpng.md 4e4c0ecd9795b9181538ea94fa63e6817e6e1f4b51481d720adaee0fde409c20 +file 444 legal/java.desktop/mesa3d.md 63f4e6f75caebbccb95d903fb43e46ac7111b3624d0a34f146b276d7d9e7b152 +file 444 legal/java.desktop/pipewire.md 56a8fb1652c70ac204d13bb52ca4d678162e7d21a97d10209c2a633de8082de0 +file 444 legal/java.desktop/xwd.md 1d4ffa93c87f35084b02a7aa90a21084b4019db4fe1003c2e5ce775b4a384f59 +directory 755 legal/java.instrument +file 444 legal/java.instrument/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.instrument/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.instrument/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.logging +file 444 legal/java.logging/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.logging/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.logging/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.management +directory 755 legal/java.management.rmi +file 444 legal/java.management.rmi/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.management.rmi/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.management.rmi/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/java.management/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.management/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.management/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.naming +file 444 legal/java.naming/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.naming/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.naming/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.net.http +file 444 legal/java.net.http/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.net.http/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.net.http/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.prefs +file 444 legal/java.prefs/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.prefs/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.prefs/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.rmi +file 444 legal/java.rmi/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.rmi/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.rmi/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.scripting +file 444 legal/java.scripting/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.scripting/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.scripting/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.se +file 444 legal/java.se/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.se/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.se/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.security.jgss +file 444 legal/java.security.jgss/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.security.jgss/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.security.jgss/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.security.sasl +file 444 legal/java.security.sasl/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.security.sasl/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.security.sasl/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.smartcardio +file 444 legal/java.smartcardio/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.smartcardio/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.smartcardio/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/java.smartcardio/pcsclite.md b39ce363c281ed36e937f9e6c03311d7dbf0b20d3614dde084130c2a10909692 +directory 755 legal/java.sql +directory 755 legal/java.sql.rowset +file 444 legal/java.sql.rowset/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.sql.rowset/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.sql.rowset/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/java.sql/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.sql/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.sql/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.transaction.xa +file 444 legal/java.transaction.xa/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.transaction.xa/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.transaction.xa/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/java.xml +directory 755 legal/java.xml.crypto +file 444 legal/java.xml.crypto/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.xml.crypto/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.xml.crypto/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/java.xml.crypto/santuario.md b7764b61731d4ee9567b090f34d02237afcfb0377e5d1136c7ad3ef345cc4937 +file 444 legal/java.xml/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/java.xml/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/java.xml/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/java.xml/bcel.md 853a1e7ce397bb10de0e2b3bde0844bcc651f17d983decd07d2d003c0304c311 +file 444 legal/java.xml/dom.md 6686e8877667584a3a7c07344baadca1a03e29f677162d87c3c0811e990d1148 +file 444 legal/java.xml/jcup.md 8d5dcfdf50455a3c34c753a98f21e953248af200415a9084e3f102cb6c43b8bf +file 444 legal/java.xml/xalan.md c27eb875da4be683d4d7422be986e5e30f636ede31958ff1d39f9cd6109e7a00 +file 444 legal/java.xml/xerces.md 4a6bf6b367193ee68681cb2d9fed30ffc5d62dd2d477bd62e0271707d71b3244 +directory 755 legal/jdk.accessibility +file 444 legal/jdk.accessibility/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.accessibility/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.accessibility/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.attach +file 444 legal/jdk.attach/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.attach/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.attach/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.charsets +file 444 legal/jdk.charsets/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.charsets/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.charsets/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.compiler +file 444 legal/jdk.compiler/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.compiler/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.compiler/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.crypto.cryptoki +file 444 legal/jdk.crypto.cryptoki/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.crypto.cryptoki/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.crypto.cryptoki/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.crypto.cryptoki/pkcs11cryptotoken.md 1f36ff1342a581142c858f90064e20633d43529ac82adb85345bd902a14e18b2 +file 444 legal/jdk.crypto.cryptoki/pkcs11wrapper.md 371974b1fca3744a3892c7ee1fcc593b8b4281fc218f4cafd2f709e9df5fd81d +directory 755 legal/jdk.crypto.ec +file 444 legal/jdk.crypto.ec/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.crypto.ec/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.crypto.ec/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.dynalink +file 444 legal/jdk.dynalink/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.dynalink/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.dynalink/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.dynalink/dynalink.md 17312591cabee3ef6c34ed8897d92e4e361ba9cea41ec00dcd61a322a8fc2cdb +directory 755 legal/jdk.editpad +file 444 legal/jdk.editpad/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.editpad/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.editpad/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.hotspot.agent +file 444 legal/jdk.hotspot.agent/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.hotspot.agent/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.hotspot.agent/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.httpserver +file 444 legal/jdk.httpserver/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.httpserver/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.httpserver/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.incubator.vector +file 444 legal/jdk.incubator.vector/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.incubator.vector/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.incubator.vector/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.internal.ed +file 444 legal/jdk.internal.ed/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.internal.ed/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.internal.ed/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.internal.jvmstat +file 444 legal/jdk.internal.jvmstat/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.internal.jvmstat/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.internal.jvmstat/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.internal.le +file 444 legal/jdk.internal.le/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.internal.le/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.internal.le/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.internal.le/jline.md aeb64d60a80564e8118799fb00b94d6686c1fe04da2f7399f970f968d32bcf84 +directory 755 legal/jdk.internal.opt +file 444 legal/jdk.internal.opt/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.internal.opt/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.internal.opt/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.internal.opt/jopt-simple.md 99bc67f93cf57d6d20e6047731c93fbb267d70fbdd4115d119e0f85c6efe5c05 +directory 755 legal/jdk.internal.vm.ci +file 444 legal/jdk.internal.vm.ci/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.internal.vm.ci/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.internal.vm.ci/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.internal.vm.compiler +directory 755 legal/jdk.internal.vm.compiler.management +file 444 legal/jdk.internal.vm.compiler.management/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.internal.vm.compiler.management/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.internal.vm.compiler.management/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.internal.vm.compiler/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.internal.vm.compiler/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.internal.vm.compiler/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jartool +file 444 legal/jdk.jartool/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jartool/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jartool/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.javadoc +file 444 legal/jdk.javadoc/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.javadoc/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.javadoc/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.javadoc/jquery.md b68f454c2bd58959c862d94bf0f16c3f78a2d537388ca060d354344db80ee695 +file 444 legal/jdk.javadoc/jqueryUI.md bb0a0e89ebd824df714516bf64b9101c62081e4b376f00f929a58c09555bf111 +directory 755 legal/jdk.jcmd +file 444 legal/jdk.jcmd/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jcmd/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jcmd/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jconsole +file 444 legal/jdk.jconsole/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jconsole/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jconsole/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jdeps +file 444 legal/jdk.jdeps/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jdeps/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jdeps/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jdi +file 444 legal/jdk.jdi/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jdi/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jdi/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jdwp.agent +file 444 legal/jdk.jdwp.agent/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jdwp.agent/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jdwp.agent/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jfr +file 444 legal/jdk.jfr/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jfr/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jfr/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jlink +file 444 legal/jdk.jlink/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jlink/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jlink/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jpackage +file 444 legal/jdk.jpackage/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jpackage/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jpackage/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jshell +file 444 legal/jdk.jshell/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jshell/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jshell/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jsobject +file 444 legal/jdk.jsobject/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jsobject/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jsobject/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.jstatd +file 444 legal/jdk.jstatd/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.jstatd/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.jstatd/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.localedata +file 444 legal/jdk.localedata/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.localedata/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.localedata/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.localedata/cldr.md 19515e14a240e022640e95b61b5095127fa9690755950b4a2b0a02e783e08163 +file 444 legal/jdk.localedata/thaidict.md c326144a2351c9608fa708b5d7d3c5a3da03e82b66479b128e9db4969539824a +directory 755 legal/jdk.management +directory 755 legal/jdk.management.agent +file 444 legal/jdk.management.agent/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.management.agent/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.management.agent/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.management.jfr +file 444 legal/jdk.management.jfr/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.management.jfr/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.management.jfr/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.management/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.management/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.management/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.naming.dns +file 444 legal/jdk.naming.dns/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.naming.dns/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.naming.dns/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.naming.rmi +file 444 legal/jdk.naming.rmi/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.naming.rmi/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.naming.rmi/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.net +file 444 legal/jdk.net/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.net/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.net/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.nio.mapmode +file 444 legal/jdk.nio.mapmode/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.nio.mapmode/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.nio.mapmode/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.random +file 444 legal/jdk.random/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.random/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.random/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.sctp +file 444 legal/jdk.sctp/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.sctp/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.sctp/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.security.auth +file 444 legal/jdk.security.auth/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.security.auth/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.security.auth/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.security.jgss +file 444 legal/jdk.security.jgss/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.security.jgss/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.security.jgss/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.unsupported +directory 755 legal/jdk.unsupported.desktop +file 444 legal/jdk.unsupported.desktop/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.unsupported.desktop/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.unsupported.desktop/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +file 444 legal/jdk.unsupported/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.unsupported/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.unsupported/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.xml.dom +file 444 legal/jdk.xml.dom/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.xml.dom/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.xml.dom/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 legal/jdk.zipfs +file 444 legal/jdk.zipfs/ADDITIONAL_LICENSE_INFO a69bce275ba7a3570af6579cb0f55682cd75fedfcd49e0e8e9022270c447c916 +file 444 legal/jdk.zipfs/ASSEMBLY_EXCEPTION 75292f03bf23d3db7c985aecc191029b93883200721ed23ed34a2e601463df33 +file 444 legal/jdk.zipfs/LICENSE 4b9abebc4338048a7c2dc184e9f800deb349366bdf28eb23c2677a77b4c87726 +directory 755 lib +file 644 lib/classlist f0d9b96c9a4eb4c068588d050734e3d25b6f10a336cd30cfd2154c8b90469408 +file 644 lib/ct.sym 4b968bfd8c02fd637566b5adb26f7872acd7d61bd0d668c056a3a332e345c271 +file 644 lib/fontconfig.bfc 5409952a218126c5c11cf0e4963665711aeeab50f5648264d70866fca705a950 +file 644 lib/fontconfig.properties.src c8d20a4a39240665252306bdd3dbc8d71e1e246c8cd8c7985b32e4101658a148 +directory 755 lib/jfr +file 644 lib/jfr/default.jfc b1765d56a0ecc57133600b7284bb7b3b977366c4365b7acffd86f8157736d53e +file 644 lib/jfr/profile.jfc 485fb90dbecee9a950c45247464351162b1eb0c35387fbb929f002c600807251 +file 644 lib/jrt-fs.jar ca81222a25fccfd0ecc9ef5c3d1efee0634e7938ab3c245e69b5bc0a55019028 +file 755 lib/jspawnhelper 9763041a25ed031e15a1b7219c4defbb86d12cdfdf18676e0f0576cba8f3e22b +file 644 lib/jvm.cfg aa9efb969444c1484e29adecab55a122458090616e766b2f1230ef05bc3867e0 +file 644 lib/libattach.dylib 9cc02a9cfdaca2ffb7118d1cac0b1dbac54f5c3c7b082156475493ccfe2a78db +file 644 lib/libawt.dylib 5f8046a4842af36861fb82e0c2c38750b31bc82ffdcf5b5fee039417800dbbae +file 644 lib/libawt_lwawt.dylib 54ba19e3f28a7198df5176b0b0a1b5a18509a2313867af5e907675f14fd1fa76 +file 644 lib/libdt_socket.dylib ecb54b15143cc89a480293b2f2071422393add941e5607114290537d3fcc1d62 +file 644 lib/libextnet.dylib fdb2e8298727673d40af55c3b6fe1609c7f62d24d7ad9dd7825c95d14edfa7a0 +file 644 lib/libfontmanager.dylib b29e851a7c0051997a93e6ce24014ddf76cb8450db17e5e66add42d4cb5c3fb3 +file 644 lib/libfreetype.dylib 51d53aaa5e0407a0894d9bdd131c93be0d65dc148991dd07aff727fe982fefab +file 644 lib/libinstrument.dylib 1f956c331d773df3b7d2e60c2f35c815705817300e8317894c5581e70715fdbd +file 644 lib/libj2gss.dylib 5fe2246ada71d8734211d37b650024be8e95aca44a7581b078aa766dade35f9e +file 644 lib/libj2pcsc.dylib 1468c674f362210ab3411e6e93b739b445b5ae21eca569f093d70f1204c0383a +file 644 lib/libj2pkcs11.dylib 69cf7e2d4610a48a7971419393918dd791fc802668d041b7fd24770ef2d43176 +file 644 lib/libjaas.dylib a446f20f0110460b9950824d0939ac27e98e5cc57b98f6792d0b1979173c4e80 +file 644 lib/libjava.dylib 641f117fe005949c2dc2778633e677bd3e57495b7e856a48c1cdd122fedfefb9 +file 644 lib/libjavajpeg.dylib 4a842dd41985fb4b6cdc3c14d86afe26e32a3eb7611658173127ae6318b0f3d7 +file 644 lib/libjawt.dylib d286340483e81bfc7be27e9902cc2cf768372113ba8d188952b6c902497606d9 +file 644 lib/libjdwp.dylib ae7b73cebc894a2c4e4e7d6af40eb647618415f71320d10eda4104ee6d1192da +file 644 lib/libjimage.dylib 4fc323921100cc77c7a08c730b41f03ee5808fca87f076430a8414ccab4e1c1b +file 644 lib/libjli.dylib eb3f98e699eaaccc00302a2cc146bed83d17ec6c76fa80457c97567e3aadc874 +file 644 lib/libjsig.dylib 1f5af23870eb9f93aa9b041d5493e4fb5b21e86fe0598ff92bbeef737a098ca6 +file 644 lib/libjsound.dylib 909a4ef7cb875e11a133f5b581848665f3b90ace0a6fd36a97219c90c6e38e34 +file 644 lib/liblcms.dylib 6f344ec4d44384e0eda34a1cb275b934551b83214016bab7149e22b30619e809 +file 644 lib/lible.dylib c486ed2997fdb00b0d5f5a40ebbb9f98d9f384614b7447ea7dcc62c7b348ee52 +file 644 lib/libmanagement.dylib 2a24c09e9026a26a7ef4313e8c6bf62b6488dd1172fe35241344fcb3aed9266f +file 644 lib/libmanagement_agent.dylib 60977c94be255a06abbbe2f7d0b98fe52602e4b82c1f72c3f22dad84359fd789 +file 644 lib/libmanagement_ext.dylib d2b808f0f02e1fe9616d7f401180b420853681d7a8279432b47032f96508c698 +file 644 lib/libmlib_image.dylib 090767e7fccb894701e136d11abd326cd746918cda9703e81814a0188748e85c +file 644 lib/libnet.dylib 643375d6f1494508111b261cdabd259a4318a18d83a989e2fdb22a02603d476d +file 644 lib/libnio.dylib 459d57393cd479594b81240fab92a7fe53b2cb87d4b44c1a58695475f89a3161 +file 644 lib/libosx.dylib a19364e334a6b45a822bd4d4c4cbd3505fb7ed549449570c5a61659238ca79b6 +file 644 lib/libosxapp.dylib b36726072aa2237d299a55134a40554433cfb837b022a921c91e6c929dd2b801 +file 644 lib/libosxkrb5.dylib 2e8d0274a75f3fef3443f116b04ef8aa85d9ea6b84368dbbaef81f89296d3f4f +file 644 lib/libosxsecurity.dylib f5759f2ed6736705b8238cc76aeee1fc221945835a241256487c713eb6c1538c +file 644 lib/libosxui.dylib 369033b5c5e3b007f359738082b8dd6b2e0a789b625e1e803620cfc7b2e58f22 +file 644 lib/libprefs.dylib f82efc2ae6ff2742993ca0be5dd9d11a28ed743cadc0aa618823b363d578ec8a +file 644 lib/librmi.dylib 66b2b0b7083fd14ed12abf515080c16dea4902f44c4fb6cfda17fbd69afa4324 +file 644 lib/libsaproc.dylib e680ffc243ae7c10e31483363f4a3e1f96ec5d18b1300ccba715857fe298780c +file 644 lib/libsplashscreen.dylib 0d51fbb72484d89f6121c3e2ddb41fb5da110c277e9cf0fd6bd610a75c1f6b87 +file 644 lib/libsyslookup.dylib 52486688fb1225e1a821eaa520f23e3bbd702b036e881a42b333b4d66cb901a7 +file 644 lib/libverify.dylib 5f76d9e047f287ed420f57b668f1c93c962555de76a3f198977aeca8c4df7a59 +file 644 lib/libzip.dylib cd8faa70e40f8afd298af7db46aa9b1f667d5460acb8d13fb9e63dd00d5c32b3 +file 644 lib/modules f46f9d1583250afaa5e81ea6747860345416ff155dd6dd717548f9f73f80a46c +file 644 lib/psfont.properties.ja 5a4bd51b969bf187ff86d94f4a71fdfbfa602762975fa3c73d264b4575f7c78f +file 644 lib/psfontj2d.properties 780c565d5af3ee6f68b887b75c041cdf46a0592f67012f12eeb691283e92630a +directory 755 lib/security +file 644 lib/security/blocked.certs 96572f243f31c2ef81a6e627542e596f6a9295cff3c7ae095c1b595cb1457ded +file 644 lib/security/cacerts 21763004eb7b171422dc09864abe24f991d905ed1ac24f4fc7716f1024b37f80 +file 644 lib/security/default.policy 2bd418aab30b091b136962f80be7ddf39dcba85f082a558a6993848ae65366ae +file 644 lib/security/public_suffix_list.dat 5a3a571cc2e016fee10c221036cf1d2b52ee8ba39288ef20abe2667828ca30b4 +directory 755 lib/server +file 444 lib/server/classes.jsa 2044781d03dd7dce562908555dea20f7fe66f44899fbfefa2bda0fe62d0a40bf +file 444 lib/server/classes_nocoops.jsa 6e44b3cc2ed99640b454d3b2581dd7a19313ba35c30b7c86ea63ee58c842eb7f +file 644 lib/server/libjsig.dylib 1f5af23870eb9f93aa9b041d5493e4fb5b21e86fe0598ff92bbeef737a098ca6 +file 644 lib/server/libjvm.dylib 3c5094dbbf657134ee51a72188e9cc12c43b972787dcf2ed7adb0f03107786a5 +file 644 lib/shaders.metallib 238889211531d6ef69e0256a6f1f5c2d49a78d36089d20a49d2898fcafd4901d +file 644 lib/src.zip 5440baccc6c54b18671ab9cb9bf6ffdd7698d3a85ee74faa0b47b111312f200c +file 644 lib/tzdb.dat 7be2a3adf10ff841579629f7725b15f0db41e8856527ec72847b5477073a3d29 +directory 755 man +directory 755 man/man1 +file 644 man/man1/jar.1 64dc2279c3c7969e19e914a10fe798d4a24d77c882c2e3591d17ee394ea0e4dd +file 644 man/man1/jarsigner.1 ef1e244687fa6dfa09cdde901b0ceb7cb0a1dd894f9ce9f3cd27b9e3aa2abcc2 +file 644 man/man1/java.1 ff7e75593061f512a7345f87a36853b7f6485e87c832981ede4c7dfbc9c08e6b +file 644 man/man1/javac.1 99ddfafbd4062693a6ec77e5d0e9a9fd15daaa7bcf0f80d310e1f2a2040e1c45 +file 644 man/man1/javadoc.1 f0abef4ca1aa23163b6b96f5c6eb7610ac9ac69a2b3815674049c091936c7143 +file 644 man/man1/javap.1 c45bf68738fe5d0084aeacfcb2b97bc8a76cb0a13b8a5f5c98b34370eee349a1 +file 644 man/man1/jcmd.1 ed96cabcd7babef911eb4146a06d213cee3b28ea95da367aa1bde2a8ce21f60c +file 644 man/man1/jconsole.1 5c659780acbb0db35f9590d2a0760ba878ef462bb7940ba363cb244ec461f882 +file 644 man/man1/jdb.1 780e637fb9ae6c6eadf2924ea2623e13026372652ffe594b76efdcb2d5297bce +file 644 man/man1/jdeprscan.1 c91d3daf9166652c9727bbbcb7a8d5a4b3f770864b52e0c6b6ab35ad946a6244 +file 644 man/man1/jdeps.1 7e2818ab0783f7022ced894580d1ab6871dcc9c4fe1f57f50e36dd615673f9bc +file 644 man/man1/jfr.1 748460507e2ea37ccf001e43eabac04c807dbf70fc87c174b69693ddabb59cbb +file 644 man/man1/jhsdb.1 3132969bb3cc25eccde8ddc856b5e38d03a2a9956f59d762dc455ec25f835737 +file 644 man/man1/jinfo.1 029b0d7001a5069b8b3ed880dd3b1eaab7ffe1b873fe64f931097552fa747424 +file 644 man/man1/jlink.1 981bfb28df31e0a2cd2719389e5899598725cc712b2de6c1d174a92374dfd6cd +file 644 man/man1/jmap.1 45eb16e8ac107b4b819a9ee9d93f6a54fefee1939b38c79b238604e04aadcaaa +file 644 man/man1/jmod.1 17a4d36ede6f65f0e65c8831a90c6437a215565ae6cdb49e935fd5b34a3a9e48 +file 644 man/man1/jpackage.1 6d829e365b50f04fd7c1e287b90a34f3aec4851cfd47de9075f430792c1f931e +file 644 man/man1/jps.1 ddf03b6789da63036c30a3305b16b3390b84408cd1a01abc8cdc2a7fbeaf1944 +file 644 man/man1/jrunscript.1 a1dc458e4559f6b386af190ad7ac66688468a76abc450fb92a801e36cb26bc9e +file 644 man/man1/jshell.1 3807609501a3fd04bff542d2596bda5e1f2b3a8a3f68cafdf2ff7259144294be +file 644 man/man1/jstack.1 60edf6a6c63eb2c7cb061e0ec85cf3437723b4fed422b28ec2c9881b1efae66c +file 644 man/man1/jstat.1 fdad79f8e207a3f290bfb4b6fa1ceffe32f8b7824b22f3eed45039c4c06ba5bf +file 644 man/man1/jstatd.1 ea8085b7592931897f5c7ecd151092b8c97cd0f7939cbe7b90ccc7ba64ff7ec2 +file 644 man/man1/jwebserver.1 320a8ceec121c7582914be6f9288d8bfdd1347021ecb217b6b4c521d88be88a6 +file 644 man/man1/keytool.1 717df369bf75fc46c33ce4ae0119d710996dacf05245640578b48c3ce7f1456a +file 644 man/man1/rmiregistry.1 c07e9f60b96a877c5205aba0b5cbbc37bd4c29b19b489d84ef0b257902d3020f +file 644 man/man1/serialver.1 bc4ee8d6a077e01f3967dbea5f092cf56eac043ddb184f954f8283ee96b13b4a +file 644 release 8e98b265f9a6fd3db04d2535108497897e87f1b3821270cf10ffa937463dc2ee diff --git a/test/conformance/n6/oracles/raw-java/run.sh b/test/conformance/n6/oracles/raw-java/run.sh new file mode 100755 index 0000000..63e0102 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/run.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=scripts/common.sh +source "$root/scripts/common.sh" + +[[ $# -gt 0 ]] || { + echo "usage: run.sh scan --input [--input ...] [--output file]" >&2 + exit 64 +} +/bin/bash "$root/scripts/build.sh" >/dev/null +exec "$(raw_java_java)" \ + -Dfile.encoding=UTF-8 \ + -Duser.language=en \ + -Duser.country=US \ + -Duser.timezone=UTC \ + -cp "$(raw_java_classpath)" \ + org.julialang.parquet.n6.raw.RawFooterScanner "$@" diff --git a/test/conformance/n6/oracles/raw-java/scripts/build.sh b/test/conformance/n6/oracles/raw-java/scripts/build.sh new file mode 100755 index 0000000..c695826 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/scripts/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=common.sh +source "$root/common.sh" + +/bin/bash "$root/generate.sh" --check +JAVAC=$(raw_java_javac) +mkdir -p "$RAW_JAVA_CLASSES_DIR" +find "$RAW_JAVA_CLASSES_DIR" -type f -name '*.class' -delete +sources="$RAW_JAVA_BUILD_DIR/sources.list" +find "$RAW_JAVA_GENERATED_DIR" "$RAW_JAVA_ROOT/src" \ + -type f -name '*.java' -print | LC_ALL=C sort > "$sources" +"$JAVAC" \ + --release "$RAW_JAVA_JAVA_RELEASE" \ + -encoding UTF-8 \ + -cp "$RAW_JAVA_DOWNLOAD_DIR/libthrift-$RAW_JAVA_THRIFT_VERSION.jar:$RAW_JAVA_DOWNLOAD_DIR/slf4j-api-1.7.36.jar" \ + -d "$RAW_JAVA_CLASSES_DIR" \ + @"$sources" +raw_java_classpath > "$RAW_JAVA_BUILD_DIR/classpath.txt" +printf 'Compiled %s Java class files for release %s.\n' \ + "$(find "$RAW_JAVA_CLASSES_DIR" -type f -name '*.class' | wc -l | tr -d ' ')" \ + "$RAW_JAVA_JAVA_RELEASE" diff --git a/test/conformance/n6/oracles/raw-java/scripts/common.sh b/test/conformance/n6/oracles/raw-java/scripts/common.sh new file mode 100755 index 0000000..2f7ab1f --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/scripts/common.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -euo pipefail + +RAW_JAVA_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +RAW_JAVA_REPO=$(cd "$RAW_JAVA_ROOT/../../../../.." && pwd) +RAW_JAVA_BUILD_DIR=${PARQUET_N6_RAW_JAVA_BUILD_DIR:-$RAW_JAVA_ROOT/build} +RAW_JAVA_DOWNLOAD_DIR="$RAW_JAVA_BUILD_DIR/downloads" +RAW_JAVA_GENERATED_DIR="$RAW_JAVA_BUILD_DIR/generated" +RAW_JAVA_CLASSES_DIR="$RAW_JAVA_BUILD_DIR/classes" +RAW_JAVA_JDK_DIR="$RAW_JAVA_BUILD_DIR/jdk" +RAW_JAVA_JDK_MANIFEST="$RAW_JAVA_ROOT/jdk-darwin-arm64-sequoia.manifest" +RAW_JAVA_IDL="$RAW_JAVA_REPO/thrift/parquet.thrift" +RAW_JAVA_PLAN="$RAW_JAVA_REPO/docs/dev/n6-statistics-plan.md" + +# shellcheck source=../toolchain.env +source "$RAW_JAVA_ROOT/toolchain.env" + +raw_java_sha256() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{print $1}' + else + sha256sum "$1" | awk '{print $1}' + fi +} + +raw_java_verify() { + local expected=$1 + local file=$2 + local actual + actual=$(raw_java_sha256 "$file") + [[ $actual == "$expected" ]] || { + echo "raw-java: SHA-256 mismatch for $file" >&2 + echo "raw-java: expected $expected" >&2 + echo "raw-java: actual $actual" >&2 + return 1 + } +} + +raw_java_mode() { + if stat -f '%Lp' "$1" >/dev/null 2>&1; then + stat -f '%Lp' "$1" + else + stat -c '%a' "$1" + fi +} + +raw_java_tree_manifest() { + local tree=$1 + local output=$2 + [[ -d $tree ]] || { + echo "raw-java: JDK tree is absent: $tree" >&2 + return 1 + } + : > "$output" + while IFS= read -r path; do + local relative=${path#"$tree"/} + local mode + mode=$(raw_java_mode "$path") + if [[ -L $path ]]; then + printf 'link\t%s\t%s\t%s\n' "$mode" "$relative" "$(readlink "$path")" >> "$output" + elif [[ -d $path ]]; then + printf 'directory\t%s\t%s\n' "$mode" "$relative" >> "$output" + elif [[ -f $path ]]; then + printf 'file\t%s\t%s\t%s\n' \ + "$mode" "$relative" "$(raw_java_sha256 "$path")" >> "$output" + else + echo "raw-java: unsupported entry in JDK tree: $relative" >&2 + return 1 + fi + done < <(find "$tree" -mindepth 1 -print | LC_ALL=C sort) +} + +raw_java_tree_sha256() { + local tree=$1 + local manifest + manifest=$(mktemp "${TMPDIR:-/tmp}/raw-java-jdk-tree.XXXXXX") + if ! raw_java_tree_manifest "$tree" "$manifest"; then + rm -f "$manifest" + return 1 + fi + local digest + digest=$(raw_java_sha256 "$manifest") + rm -f "$manifest" + printf '%s\n' "$digest" +} + +raw_java_verify_tree() { + local expected=$1 + local tree=$2 + raw_java_verify "$expected" "$RAW_JAVA_JDK_MANIFEST" + local actual_manifest + actual_manifest=$(mktemp "${TMPDIR:-/tmp}/raw-java-jdk-tree.XXXXXX") + if ! raw_java_tree_manifest "$tree" "$actual_manifest"; then + rm -f "$actual_manifest" + return 1 + fi + if cmp -s "$RAW_JAVA_JDK_MANIFEST" "$actual_manifest"; then + rm -f "$actual_manifest" + return + fi + local actual + actual=$(raw_java_sha256 "$actual_manifest") + rm -f "$actual_manifest" + { + echo "raw-java: full JDK tree SHA-256 mismatch for $tree" >&2 + echo "raw-java: expected $expected" >&2 + echo "raw-java: actual $actual" >&2 + } + return 1 +} + +raw_java_download() { + local url=$1 + local expected=$2 + local output=$3 + if [[ -f $output ]]; then + raw_java_verify "$expected" "$output" + return + fi + mkdir -p "$(dirname "$output")" + local temporary="$output.part" + curl --fail --location --silent --show-error --output "$temporary" "$url" + raw_java_verify "$expected" "$temporary" + mv "$temporary" "$output" +} + +raw_java_verify_authority() { + raw_java_verify "$RAW_JAVA_PLAN_SHA256" "$RAW_JAVA_PLAN" + raw_java_verify "$RAW_JAVA_IDL_SHA256" "$RAW_JAVA_IDL" +} + +raw_java_classpath() { + printf '%s:%s:%s:%s\n' \ + "$RAW_JAVA_CLASSES_DIR" \ + "$RAW_JAVA_DOWNLOAD_DIR/libthrift-$RAW_JAVA_THRIFT_VERSION.jar" \ + "$RAW_JAVA_DOWNLOAD_DIR/slf4j-api-1.7.36.jar" \ + "$RAW_JAVA_DOWNLOAD_DIR/slf4j-nop-1.7.36.jar" +} + +raw_java_java() { + printf '%s\n' "$RAW_JAVA_JDK_DIR/bin/java" +} + +raw_java_javac() { + printf '%s\n' "$RAW_JAVA_JDK_DIR/bin/javac" +} diff --git a/test/conformance/n6/oracles/raw-java/scripts/fetch-toolchain.sh b/test/conformance/n6/oracles/raw-java/scripts/fetch-toolchain.sh new file mode 100755 index 0000000..6a63a7f --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/scripts/fetch-toolchain.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=common.sh +source "$root/common.sh" + +raw_java_platform_pins() { + local system + local machine + system=$(uname -s) + machine=$(uname -m) + case "$system:$machine" in + Darwin:arm64) + local major + major=$(sw_vers -productVersion | cut -d. -f1) + [[ $major == 15 ]] || { + echo "raw-java: the exact toolchain is validated only on macOS 15 arm64" >&2 + return 1 + } + printf '%s\n' \ + "$RAW_JAVA_BOTTLE_DARWIN_ARM64_SEQUOIA_SHA256" \ + "$RAW_JAVA_COMPILER_DARWIN_ARM64_SEQUOIA_SHA256" \ + "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_URL" \ + "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_SHA256" \ + "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_JAVA_SHA256" \ + "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_JAVAC_SHA256" \ + "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_RELEASE_SHA256" \ + "$RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_TREE_SHA256" + ;; + *) + echo "raw-java: no pinned Thrift compiler for $system $machine" >&2 + return 1 + ;; + esac +} + +raw_java_fetch_bottle() { + local digest=$1 + local output=$2 + if [[ -f $output ]]; then + raw_java_verify "$digest" "$output" + return + fi + mkdir -p "$(dirname "$output")" + local token + token=$(curl --fail --silent --show-error \ + 'https://ghcr.io/token?scope=repository:homebrew/core/thrift:pull&service=ghcr.io' | + sed -E 's/.*"token":"([^"]+)".*/\1/') + [[ -n $token ]] || { + echo "raw-java: cannot obtain the public GHCR token" >&2 + return 1 + } + local temporary="$output.part" + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer $token" \ + --output "$temporary" \ + "https://ghcr.io/v2/homebrew/core/thrift/blobs/sha256:$digest" + raw_java_verify "$digest" "$temporary" + mv "$temporary" "$output" +} + +raw_java_install_compiler() { + local bottle_digest=$1 + local compiler_digest=$2 + local compiler="$RAW_JAVA_BUILD_DIR/compiler/thrift" + if [[ -x $compiler ]]; then + if [[ $(raw_java_sha256 "$compiler") == "$compiler_digest" ]] && + [[ $("$compiler" --version) == "Thrift version $RAW_JAVA_THRIFT_VERSION" ]]; then + return + fi + fi + local bottle="$RAW_JAVA_DOWNLOAD_DIR/thrift-$RAW_JAVA_THRIFT_VERSION-$bottle_digest.bottle.tar.gz" + raw_java_fetch_bottle "$bottle_digest" "$bottle" + local unpacked="$RAW_JAVA_BUILD_DIR/compiler-unpacked" + mkdir -p "$unpacked" "$RAW_JAVA_BUILD_DIR/compiler" + find "$unpacked" -mindepth 1 -delete + tar -xzf "$bottle" -C "$unpacked" + local source + source=$(find "$unpacked" -type f -path '*/bin/thrift' -print -quit) + [[ -n $source ]] || { + echo "raw-java: compiler bottle does not contain bin/thrift" >&2 + return 1 + } + cp "$source" "$compiler" + chmod 0755 "$compiler" + raw_java_verify "$compiler_digest" "$compiler" + [[ $("$compiler" --version) == "Thrift version $RAW_JAVA_THRIFT_VERSION" ]] || { + echo "raw-java: extracted compiler has the wrong version" >&2 + return 1 + } +} + +raw_java_install_jdk() { + local url=$1 + local archive_digest=$2 + local java_digest=$3 + local javac_digest=$4 + local release_digest=$5 + local tree_digest=$6 + local archive="$RAW_JAVA_DOWNLOAD_DIR/OpenJDK21U-jdk_aarch64_mac_hotspot_21.0.8_9.tar.gz" + raw_java_download "$url" "$archive_digest" "$archive" + if [[ -x $RAW_JAVA_JDK_DIR/bin/java && -x $RAW_JAVA_JDK_DIR/bin/javac && + -f $RAW_JAVA_JDK_DIR/release ]] && + raw_java_verify "$java_digest" "$RAW_JAVA_JDK_DIR/bin/java" >/dev/null 2>&1 && + raw_java_verify "$javac_digest" "$RAW_JAVA_JDK_DIR/bin/javac" >/dev/null 2>&1 && + raw_java_verify "$release_digest" "$RAW_JAVA_JDK_DIR/release" >/dev/null 2>&1 && + raw_java_verify_tree "$tree_digest" "$RAW_JAVA_JDK_DIR" >/dev/null 2>&1; then + return + fi + local unpacked="$RAW_JAVA_BUILD_DIR/jdk-unpacked" + mkdir -p "$unpacked" + find "$unpacked" -mindepth 1 -delete + tar -xzf "$archive" -C "$unpacked" + local home + home=$(find "$unpacked" -type f -path '*/bin/java' -print -quit) + [[ -n $home ]] || { + echo "raw-java: pinned JDK archive does not contain bin/java" >&2 + return 1 + } + home=${home%/bin/java} + find "$RAW_JAVA_JDK_DIR" -mindepth 1 -delete 2>/dev/null || true + mkdir -p "$RAW_JAVA_JDK_DIR" + cp -R "$home/." "$RAW_JAVA_JDK_DIR/" + raw_java_verify "$java_digest" "$RAW_JAVA_JDK_DIR/bin/java" + raw_java_verify "$javac_digest" "$RAW_JAVA_JDK_DIR/bin/javac" + raw_java_verify "$release_digest" "$RAW_JAVA_JDK_DIR/release" + raw_java_verify_tree "$tree_digest" "$RAW_JAVA_JDK_DIR" + [[ $("$RAW_JAVA_JDK_DIR/bin/javac" -version 2>&1) == "javac 21.0.8" ]] || { + echo "raw-java: extracted javac has the wrong version" >&2 + return 1 + } +} + +raw_java_verify_authority +mkdir -p "$RAW_JAVA_DOWNLOAD_DIR" +platform_pins=() +while IFS= read -r pin; do + platform_pins+=("$pin") +done < <(raw_java_platform_pins) +[[ ${#platform_pins[@]} -eq 8 ]] || { + echo "raw-java: incomplete platform toolchain pins" >&2 + exit 1 +} +raw_java_download "$RAW_JAVA_THRIFT_SOURCE_URL" \ + "$RAW_JAVA_THRIFT_SOURCE_SHA256" \ + "$RAW_JAVA_DOWNLOAD_DIR/thrift-$RAW_JAVA_THRIFT_VERSION.tar.gz" +raw_java_download "$RAW_JAVA_HOMEBREW_FORMULA_URL" \ + "$RAW_JAVA_HOMEBREW_FORMULA_SHA256" \ + "$RAW_JAVA_DOWNLOAD_DIR/homebrew-thrift-$RAW_JAVA_HOMEBREW_FORMULA_COMMIT.rb" +raw_java_download "$RAW_JAVA_LIBTHRIFT_URL" \ + "$RAW_JAVA_LIBTHRIFT_SHA256" \ + "$RAW_JAVA_DOWNLOAD_DIR/libthrift-$RAW_JAVA_THRIFT_VERSION.jar" +raw_java_download "$RAW_JAVA_LIBTHRIFT_POM_URL" \ + "$RAW_JAVA_LIBTHRIFT_POM_SHA256" \ + "$RAW_JAVA_DOWNLOAD_DIR/libthrift-$RAW_JAVA_THRIFT_VERSION.pom" +raw_java_download "$RAW_JAVA_SLF4J_API_URL" \ + "$RAW_JAVA_SLF4J_API_SHA256" \ + "$RAW_JAVA_DOWNLOAD_DIR/slf4j-api-1.7.36.jar" +raw_java_download "$RAW_JAVA_SLF4J_NOP_URL" \ + "$RAW_JAVA_SLF4J_NOP_SHA256" \ + "$RAW_JAVA_DOWNLOAD_DIR/slf4j-nop-1.7.36.jar" +raw_java_install_compiler "${platform_pins[0]}" "${platform_pins[1]}" +raw_java_install_jdk "${platform_pins[2]}" "${platform_pins[3]}" \ + "${platform_pins[4]}" "${platform_pins[5]}" "${platform_pins[6]}" \ + "${platform_pins[7]}" +printf 'Thrift compiler: %s\n' "$RAW_JAVA_THRIFT_VERSION" +printf 'Thrift runtime: %s\n' "$RAW_JAVA_THRIFT_VERSION" +printf 'Java toolchain: %s %s\n' "$RAW_JAVA_JDK_VENDOR" "$RAW_JAVA_JDK_VERSION" diff --git a/test/conformance/n6/oracles/raw-java/scripts/generate.sh b/test/conformance/n6/oracles/raw-java/scripts/generate.sh new file mode 100755 index 0000000..6f17a5d --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/scripts/generate.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=common.sh +source "$root/common.sh" + +if [[ $# -gt 1 || ( $# -eq 1 && $1 != "--check" ) ]]; then + echo "usage: generate.sh [--check]" >&2 + exit 64 +fi + +/bin/bash "$root/fetch-toolchain.sh" >/dev/null +mkdir -p "$RAW_JAVA_GENERATED_DIR" +find "$RAW_JAVA_GENERATED_DIR" -type f -name '*.java' -delete +"$RAW_JAVA_BUILD_DIR/compiler/thrift" \ + --gen java:generated_annotations=suppress \ + -out "$RAW_JAVA_GENERATED_DIR" \ + "$RAW_JAVA_IDL" + +manifest="$RAW_JAVA_BUILD_DIR/generated.sha256" +: > "$manifest" +while IFS= read -r file; do + relative=${file#"$RAW_JAVA_GENERATED_DIR/"} + printf '%s %s\n' "$(raw_java_sha256 "$file")" "$relative" >> "$manifest" +done < <(find "$RAW_JAVA_GENERATED_DIR" -type f -name '*.java' -print | LC_ALL=C sort) +raw_java_verify "$RAW_JAVA_GENERATED_MANIFEST_SHA256" "$manifest" +printf 'Generated %s pinned Java format sources.\n' \ + "$(find "$RAW_JAVA_GENERATED_DIR" -type f -name '*.java' | wc -l | tr -d ' ')" diff --git a/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/CompactThriftPreflight.java b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/CompactThriftPreflight.java new file mode 100644 index 0000000..40e1044 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/CompactThriftPreflight.java @@ -0,0 +1,291 @@ +package org.julialang.parquet.n6.raw; + +import java.io.IOException; + +final class CompactThriftPreflight { + private static final int BOOLEAN_TRUE = 1; + private static final int BOOLEAN_FALSE = 2; + private static final int BYTE = 3; + private static final int I16 = 4; + private static final int I32 = 5; + private static final int I64 = 6; + private static final int DOUBLE = 7; + private static final int BINARY = 8; + private static final int LIST = 9; + private static final int SET = 10; + private static final int MAP = 11; + private static final int STRUCT = 12; + private static final int UUID = 13; + private static final int MAXIMUM_DEPTH = 128; + private static final long MAXIMUM_CONTAINER_ELEMENTS = 1_000_000; + private static final long MAXIMUM_AGGREGATE_ELEMENTS = 4_000_000; + private static final long MAXIMUM_BINARY_BYTES = 64L * 1024 * 1024; + private static final long MAXIMUM_AGGREGATE_BYTES = 64L * 1024 * 1024; + + private final byte[] bytes; + private final int maximumDepth; + private final long maximumContainerElements; + private final long maximumAggregateElements; + private final long maximumBinaryBytes; + private final long maximumAggregateBytes; + private int position; + private long aggregateElements; + private long aggregateBytes; + + private CompactThriftPreflight(byte[] bytes) { + this(bytes, MAXIMUM_DEPTH, MAXIMUM_CONTAINER_ELEMENTS, MAXIMUM_AGGREGATE_ELEMENTS, + MAXIMUM_BINARY_BYTES, MAXIMUM_AGGREGATE_BYTES); + } + + private CompactThriftPreflight( + byte[] bytes, + int maximumDepth, + long maximumContainerElements, + long maximumAggregateElements, + long maximumBinaryBytes, + long maximumAggregateBytes + ) { + if (maximumDepth < 0 || maximumContainerElements < 0 || maximumAggregateElements < 0 + || maximumBinaryBytes < 0 || maximumAggregateBytes < 0) { + throw new IllegalArgumentException("Compact-Thrift limits must be nonnegative"); + } + this.bytes = bytes; + this.maximumDepth = maximumDepth; + this.maximumContainerElements = maximumContainerElements; + this.maximumAggregateElements = maximumAggregateElements; + this.maximumBinaryBytes = maximumBinaryBytes; + this.maximumAggregateBytes = maximumAggregateBytes; + } + + static void validate(byte[] bytes) throws IOException { + CompactThriftPreflight preflight = new CompactThriftPreflight(bytes); + preflight.validate(); + } + + static void validateForTest( + byte[] bytes, + int maximumDepth, + long maximumContainerElements, + long maximumAggregateElements, + long maximumBinaryBytes, + long maximumAggregateBytes + ) throws IOException { + CompactThriftPreflight preflight = new CompactThriftPreflight( + bytes, maximumDepth, maximumContainerElements, maximumAggregateElements, + maximumBinaryBytes, maximumAggregateBytes); + preflight.validate(); + } + + private void validate() throws IOException { + readStruct(0); + if (position != bytes.length) { + throw new IOException("Compact-Thrift footer has trailing bytes after structural preflight"); + } + } + + private void readStruct(int depth) throws IOException { + requireDepth(depth); + int lastFieldId = 0; + while (true) { + int header = readUnsignedByte(); + if (header == 0) { + return; + } + int type = header & 0x0f; + requireType(type, false); + int delta = header >>> 4; + int fieldId; + if (delta == 0) { + long encoded = readUnsignedVarint32(); + if (encoded > 0xffffL) { + throw new IOException("Compact-Thrift field ID is outside i16 range"); + } + int value = (int) encoded; + fieldId = (value >>> 1) ^ -(value & 1); + } else { + fieldId = lastFieldId + delta; + if (fieldId > Short.MAX_VALUE) { + throw new IOException("Compact-Thrift delta field ID is outside i16 range"); + } + } + lastFieldId = fieldId; + readFieldValue(type, depth); + } + } + + private void readFieldValue(int type, int depth) throws IOException { + if (type == BOOLEAN_TRUE || type == BOOLEAN_FALSE) { + return; + } + readValue(type, depth); + } + + private void readValue(int type, int depth) throws IOException { + switch (type) { + case BOOLEAN_TRUE: + case BOOLEAN_FALSE: + int booleanValue = readUnsignedByte(); + if (booleanValue != BOOLEAN_TRUE && booleanValue != BOOLEAN_FALSE) { + throw new IOException("Compact-Thrift collection contains an invalid boolean"); + } + return; + case BYTE: + skip(1); + return; + case I16: + case I32: + readUnsignedVarint32(); + return; + case I64: + readUnsignedVarint64(); + return; + case DOUBLE: + skip(8); + return; + case BINARY: + readBinary(); + return; + case LIST: + case SET: + readList(depth + 1); + return; + case MAP: + readMap(depth + 1); + return; + case STRUCT: + readStruct(depth + 1); + return; + case UUID: + skip(16); + return; + default: + throw new IOException("unsupported Compact-Thrift type: " + type); + } + } + + private void readBinary() throws IOException { + long length = readUnsignedVarint32(); + if (length > Integer.MAX_VALUE || length > maximumBinaryBytes) { + throw new IOException("Compact-Thrift binary length exceeds the scanner limit: " + length); + } + addAggregateBytes(length); + skip((int) length); + } + + private void readList(int depth) throws IOException { + requireDepth(depth); + int header = readUnsignedByte(); + long size = header >>> 4; + int type = header & 0x0f; + requireType(type, true); + if (size == 15) { + size = readUnsignedVarint32(); + } + requireContainer(size, 1); + for (long index = 0; index < size; index++) { + readValue(type, depth); + } + } + + private void readMap(int depth) throws IOException { + requireDepth(depth); + long size = readUnsignedVarint32(); + requireContainer(size, 2); + if (size == 0) { + return; + } + int types = readUnsignedByte(); + int keyType = types >>> 4; + int valueType = types & 0x0f; + requireType(keyType, true); + requireType(valueType, true); + for (long index = 0; index < size; index++) { + readValue(keyType, depth); + readValue(valueType, depth); + } + } + + private void requireContainer(long size, int valuesPerElement) throws IOException { + if (size > Integer.MAX_VALUE || size > maximumContainerElements) { + throw new IOException("Compact-Thrift container length exceeds the scanner limit: " + size); + } + long values; + try { + values = Math.multiplyExact(size, valuesPerElement); + aggregateElements = Math.addExact(aggregateElements, values); + } catch (ArithmeticException exception) { + throw new IOException("Compact-Thrift aggregate element count overflows", exception); + } + if (aggregateElements > maximumAggregateElements) { + throw new IOException( + "Compact-Thrift aggregate element count exceeds the scanner limit: " + aggregateElements); + } + } + + private void addAggregateBytes(long length) throws IOException { + try { + aggregateBytes = Math.addExact(aggregateBytes, length); + } catch (ArithmeticException exception) { + throw new IOException("Compact-Thrift aggregate binary byte count overflows", exception); + } + if (aggregateBytes > maximumAggregateBytes) { + throw new IOException( + "Compact-Thrift aggregate binary bytes exceed the scanner limit: " + aggregateBytes); + } + } + + private void requireDepth(int depth) throws IOException { + if (depth > maximumDepth) { + throw new IOException("Compact-Thrift nesting exceeds the scanner limit: " + depth); + } + } + + private static void requireType(int type, boolean collection) throws IOException { + boolean valid = type >= BOOLEAN_TRUE && type <= UUID; + if (!valid || (!collection && type == 0)) { + throw new IOException("Compact-Thrift value has an invalid type: " + type); + } + } + + private long readUnsignedVarint32() throws IOException { + long value = 0; + for (int index = 0; index < 5; index++) { + int next = readUnsignedByte(); + if (index == 4 && (next & 0xf0) != 0) { + throw new IOException("Compact-Thrift varint32 overflows"); + } + value |= (long) (next & 0x7f) << (index * 7); + if ((next & 0x80) == 0) { + return value; + } + } + throw new IOException("Compact-Thrift varint32 is unterminated"); + } + + private void readUnsignedVarint64() throws IOException { + for (int index = 0; index < 10; index++) { + int next = readUnsignedByte(); + if (index == 9 && (next & 0xfe) != 0) { + throw new IOException("Compact-Thrift varint64 overflows"); + } + if ((next & 0x80) == 0) { + return; + } + } + throw new IOException("Compact-Thrift varint64 is unterminated"); + } + + private int readUnsignedByte() throws IOException { + if (position >= bytes.length) { + throw new IOException("Compact-Thrift footer ends inside a value"); + } + return bytes[position++] & 0xff; + } + + private void skip(int count) throws IOException { + if (count < 0 || count > bytes.length - position) { + throw new IOException("Compact-Thrift declared value exceeds the footer envelope"); + } + position += count; + } +} diff --git a/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/JsonWriter.java b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/JsonWriter.java new file mode 100644 index 0000000..551c54f --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/JsonWriter.java @@ -0,0 +1,236 @@ +package org.julialang.parquet.n6.raw; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Deque; + +final class JsonWriter { + static final class LimitException extends IllegalStateException { + private LimitException() { + super("raw footer evidence exceeds the scanner memory limit"); + } + } + + private static final class Context { + private final boolean object; + private boolean first = true; + private boolean awaitingValue; + + private Context(boolean object) { + this.object = object; + } + } + + private final StringBuilder output = new StringBuilder(); + private final Deque contexts = new ArrayDeque<>(); + private final long maximumCharacters; + private boolean rootWritten; + + JsonWriter(long maximumCharacters) { + if (maximumCharacters < 0) { + throw new LimitException(); + } + this.maximumCharacters = maximumCharacters; + } + + JsonWriter beginObject() { + beforeValue(); + append('{'); + contexts.push(new Context(true)); + return this; + } + + JsonWriter endObject() { + Context context = requireContext(true); + if (context.awaitingValue) { + throw new IllegalStateException("JSON object name has no value"); + } + contexts.pop(); + append('}'); + return this; + } + + JsonWriter beginArray() { + beforeValue(); + append('['); + contexts.push(new Context(false)); + return this; + } + + JsonWriter endArray() { + requireContext(false); + contexts.pop(); + append(']'); + return this; + } + + JsonWriter name(String name) { + Context context = requireContext(true); + if (context.awaitingValue) { + throw new IllegalStateException("JSON object name has no value"); + } + if (!context.first) { + append(','); + } + context.first = false; + appendString(name); + append(':'); + context.awaitingValue = true; + return this; + } + + JsonWriter value(String value) { + if (value == null) { + return nullValue(); + } + beforeValue(); + appendString(value); + return this; + } + + JsonWriter value(long value) { + beforeValue(); + append(Long.toString(value)); + return this; + } + + JsonWriter value(boolean value) { + beforeValue(); + append(value ? "true" : "false"); + return this; + } + + JsonWriter nullValue() { + beforeValue(); + append("null"); + return this; + } + + JsonWriter hexValue(ByteBuffer bytes, boolean reverse, boolean prefix) { + beforeValue(); + int first = bytes.position(); + int length = bytes.remaining(); + long characters = 2L + length * 2L + (prefix ? 2L : 0L); + requireCharacters(characters); + output.append('"'); + if (prefix) { + output.append("0x"); + } + if (reverse) { + for (int index = first + length - 1; index >= first; index--) { + appendHexByteUnchecked(bytes.get(index)); + } + } else { + for (int index = first; index < first + length; index++) { + appendHexByteUnchecked(bytes.get(index)); + } + } + output.append('"'); + return this; + } + + String finish() { + if (!rootWritten || !contexts.isEmpty()) { + throw new IllegalStateException("JSON document is incomplete"); + } + return output.toString(); + } + + private Context requireContext(boolean object) { + Context context = contexts.peek(); + if (context == null || context.object != object) { + throw new IllegalStateException("JSON container mismatch"); + } + return context; + } + + private void beforeValue() { + Context context = contexts.peek(); + if (context == null) { + if (rootWritten) { + throw new IllegalStateException("JSON document has multiple roots"); + } + rootWritten = true; + return; + } + if (context.object) { + if (!context.awaitingValue) { + throw new IllegalStateException("JSON object value has no name"); + } + context.awaitingValue = false; + return; + } + if (!context.first) { + append(','); + } + context.first = false; + } + + private void appendString(String value) { + append('"'); + for (int index = 0; index < value.length(); index++) { + char character = value.charAt(index); + switch (character) { + case '"': + append("\\\""); + break; + case '\\': + append("\\\\"); + break; + case '\b': + append("\\b"); + break; + case '\f': + append("\\f"); + break; + case '\n': + append("\\n"); + break; + case '\r': + append("\\r"); + break; + case '\t': + append("\\t"); + break; + default: + if (character < 0x20 || character > 0x7e) { + append("\\u"); + appendHexDigit(character >>> 12); + appendHexDigit(character >>> 8); + appendHexDigit(character >>> 4); + appendHexDigit(character); + } else { + append(character); + } + break; + } + } + append('"'); + } + + private void appendHexDigit(int value) { + append("0123456789abcdef".charAt(value & 0x0f)); + } + + private void appendHexByteUnchecked(byte value) { + int unsigned = value & 0xff; + output.append("0123456789abcdef".charAt(unsigned >>> 4)); + output.append("0123456789abcdef".charAt(unsigned & 0x0f)); + } + + private void append(char value) { + requireCharacters(1); + output.append(value); + } + + private void append(String value) { + requireCharacters(value.length()); + output.append(value); + } + + private void requireCharacters(long additional) { + if (additional > maximumCharacters - output.length()) { + throw new LimitException(); + } + } +} diff --git a/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/RawFooterScanner.java b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/RawFooterScanner.java new file mode 100644 index 0000000..916a2df --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/RawFooterScanner.java @@ -0,0 +1,1274 @@ +package org.julialang.parquet.n6.raw; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.parquet.format.ColumnChunk; +import org.apache.parquet.format.ColumnMetaData; +import org.apache.parquet.format.DecimalType; +import org.apache.parquet.format.FileMetaData; +import org.apache.parquet.format.GeographyType; +import org.apache.parquet.format.GeometryType; +import org.apache.parquet.format.IntType; +import org.apache.parquet.format.LogicalType; +import org.apache.parquet.format.RowGroup; +import org.apache.parquet.format.SchemaElement; +import org.apache.parquet.format.Statistics; +import org.apache.parquet.format.TimeType; +import org.apache.parquet.format.TimeUnit; +import org.apache.parquet.format.TimestampType; +import org.apache.parquet.format.Type; +import org.apache.parquet.format.VariantType; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TMap; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TSet; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.protocol.TType; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TMemoryInputTransport; + +public final class RawFooterScanner { + private static final byte[] MAGIC = new byte[] {'P', 'A', 'R', '1'}; + private static final String EVIDENCE_VERSION = "parquet-2.13-raw-footer-v3"; + private static final String FORMAT_COMMIT = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0"; + private static final String THRIFT_VERSION = "0.23.0"; + private static final long MAX_FILE_BYTES = 8L * 1024 * 1024 * 1024; + private static final int MAX_FOOTER_BYTES = 64 * 1024 * 1024; + private static final int MAX_INPUT_FILES = 10_000; + private static final int MAX_SCHEMA_ELEMENTS = 1_000_000; + private static final long MAX_THRIFT_BINARY_BYTES = 64L * 1024 * 1024; + private static final long MAX_THRIFT_CONTAINER_ELEMENTS = 1_000_000; + private static final int MAX_THRIFT_DEPTH = 128; + private static final long MAX_EVIDENCE_CHARACTERS = 256L * 1024 * 1024; + + private static final class InputFile { + private final String label; + private final Path path; + + private InputFile(String label, Path path) { + this.label = label; + this.path = path; + } + } + static final class Footer { + final long fileSize; + final int footerLength; + final byte[] footerBytes; + final String fileSha256; + final FileMetaData metadata; + final RawColumnOrders rawColumnOrders; + + Footer( + long fileSize, + int footerLength, + byte[] footerBytes, + String fileSha256, + FileMetaData metadata, + RawColumnOrders rawColumnOrders + ) { + this.fileSize = fileSize; + this.footerLength = footerLength; + this.footerBytes = footerBytes; + this.fileSha256 = fileSha256; + this.metadata = metadata; + this.rawColumnOrders = rawColumnOrders; + } + } + + static final class RawColumnOrder { + final Integer fieldId; + final Byte wireType; + final String headerHex; + final String state; + final String member; + final int headerOffset; + + RawColumnOrder( + Integer fieldId, + Byte wireType, + String headerHex, + String state, + String member, + int headerOffset + ) { + this.fieldId = fieldId; + this.wireType = wireType; + this.headerHex = headerHex; + this.state = state; + this.member = member; + this.headerOffset = headerOffset; + } + } + + static final class RawColumnOrders { + final boolean present; + final List values; + + RawColumnOrders(boolean present, List values) { + this.present = present; + this.values = values; + } + } + + private static final class SchemaFrame { + private int remaining; + private final List path; + + private SchemaFrame(int remaining, List path) { + this.remaining = remaining; + this.path = path; + } + } + + private static final class Leaf { + private final int ordinal; + private final List path; + private final SchemaElement schema; + + private Leaf(int ordinal, List path, SchemaElement schema) { + this.ordinal = ordinal; + this.path = path; + this.schema = schema; + } + } + + private RawFooterScanner() { + } + + public static void main(String[] arguments) throws Exception { + execute(arguments); + } + + static void execute(String[] arguments) throws Exception { + execute(arguments, MAX_EVIDENCE_CHARACTERS); + } + + static void execute(String[] arguments, long maximumEvidenceCharacters) throws Exception { + if (maximumEvidenceCharacters < 0) { + throw new IOException("raw footer evidence exceeds the scanner memory limit"); + } + if (arguments.length == 0 || !"scan".equals(arguments[0])) { + throw new IllegalArgumentException( + "usage: RawFooterScanner scan --input [--input ...] [--output file]"); + } + List inputs = new ArrayList<>(); + Path output = null; + for (int index = 1; index < arguments.length; index++) { + String argument = arguments[index]; + if ("--input".equals(argument)) { + index++; + requireArgument(arguments, index, "--input"); + inputs.add(Path.of(arguments[index])); + } else if ("--output".equals(argument)) { + if (output != null) { + throw new IllegalArgumentException("--output may be specified only once"); + } + index++; + requireArgument(arguments, index, "--output"); + output = Path.of(arguments[index]); + } else { + throw new IllegalArgumentException("unknown argument: " + argument); + } + } + if (inputs.isEmpty()) { + throw new IllegalArgumentException("at least one --input is required"); + } + List files = collectInputs(inputs); + validateOutput(files, output); + List evidence = new ArrayList<>(files.size()); + long evidenceCharacters = 0; + for (InputFile file : files) { + long remaining = maximumEvidenceCharacters - evidenceCharacters; + if (remaining <= 1) { + throw new IOException("raw footer evidence exceeds the scanner memory limit"); + } + String record = scan(file.path, file.label, remaining - 1); + evidenceCharacters = Math.addExact(evidenceCharacters, record.length() + 1L); + evidence.add(record); + } + validateOutput(files, output); + writeEvidence(evidence, output); + } + + static String scan(Path path, String label) throws IOException, TException { + return scan(path, label, MAX_EVIDENCE_CHARACTERS - 1); + } + + static String scan(Path path, String label, long maximumCharacters) throws IOException, TException { + Footer footer = readFooter(path); + List leaves = schemaLeaves(footer.metadata); + Map, Leaf> leavesByPath = new HashMap<>(); + for (Leaf leaf : leaves) { + if (leavesByPath.put(leaf.path, leaf) != null) { + throw new IOException("schema contains a duplicate leaf path: " + leaf.path); + } + } + validateRowGroups(footer.metadata, leaves, leavesByPath); + try { + JsonWriter json = new JsonWriter(maximumCharacters); + json.beginObject(); + json.name("evidence_version").value(EVIDENCE_VERSION); + json.name("format_commit").value(FORMAT_COMMIT); + json.name("thrift_version").value(THRIFT_VERSION); + json.name("file").value(label); + json.name("file_size").value(footer.fileSize); + json.name("file_sha256").value(footer.fileSha256); + json.name("footer_length").value(footer.footerLength); + json.name("footer_sha256").value(sha256(footer.footerBytes)); + appendMetadata(json, footer, leaves); + json.endObject(); + return json.finish(); + } catch (JsonWriter.LimitException exception) { + throw new IOException(exception.getMessage(), exception); + } + } + + static Footer readFooter(Path path) throws IOException, TException { + return readFooter(path, null); + } + + static Footer readFooter(Path path, Runnable betweenSnapshots) throws IOException, TException { + BasicFileAttributes before = Files.readAttributes(path, BasicFileAttributes.class); + Footer result; + try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "r")) { + long fileSize = file.length(); + if (fileSize < 12) { + throw new IOException("Parquet file is shorter than the minimum footer envelope: " + path); + } + if (fileSize > MAX_FILE_BYTES) { + throw new IOException("Parquet file exceeds the scanner size limit: " + fileSize); + } + byte[] marker = new byte[4]; + file.seek(0); + file.readFully(marker); + requireMagic(marker, "leading", path); + file.seek(fileSize - 4); + file.readFully(marker); + requireMagic(marker, "trailing", path); + byte[] lengthBytes = new byte[4]; + file.seek(fileSize - 8); + file.readFully(lengthBytes); + long unsignedLength = unsignedLittleEndianInt(lengthBytes); + long footerStart; + try { + footerStart = Math.subtractExact(Math.subtractExact(fileSize, 8L), unsignedLength); + } catch (ArithmeticException exception) { + throw new IOException("Parquet footer length overflows its file envelope: " + path, exception); + } + if (footerStart < 4) { + throw new IOException("Parquet footer length is outside its file envelope: " + path); + } + if (unsignedLength > MAX_FOOTER_BYTES) { + throw new IOException("Parquet footer exceeds the scanner size limit: " + unsignedLength); + } + String firstFileHash = sha256(file, fileSize); + byte[] footerBytes = new byte[(int) unsignedLength]; + file.seek(footerStart); + file.readFully(footerBytes); + CompactThriftPreflight.validate(footerBytes); + RawColumnOrders rawColumnOrders = readRawColumnOrders(footerBytes); + byte[] semanticBytes = withoutColumnOrders(footerBytes); + TMemoryInputTransport transport = new TMemoryInputTransport(semanticBytes); + FileMetaData metadata = new FileMetaData(); + metadata.read(new TCompactProtocol( + transport, MAX_THRIFT_BINARY_BYTES, MAX_THRIFT_CONTAINER_ELEMENTS)); + int trailing = transport.getBytesRemainingInBuffer(); + if (trailing != 0) { + throw new IOException("Compact-Thrift footer has " + trailing + " trailing byte(s): " + path); + } + if (betweenSnapshots != null) { + betweenSnapshots.run(); + } + if (file.length() != fileSize) { + throw new IOException("Parquet input changed size during the scan: " + path); + } + String secondFileHash = sha256(file, fileSize); + if (!firstFileHash.equals(secondFileHash)) { + throw new IOException("Parquet input changed during the scan: " + path); + } + result = new Footer(fileSize, (int) unsignedLength, footerBytes, + firstFileHash, metadata, rawColumnOrders); + } + BasicFileAttributes after = Files.readAttributes(path, BasicFileAttributes.class); + if (!sameFileSnapshot(before, after)) { + throw new IOException("Parquet input path changed during the scan: " + path); + } + return result; + } + + private static void appendMetadata( + JsonWriter json, + Footer footer, + List leaves + ) { + FileMetaData metadata = footer.metadata; + json.name("file_metadata").beginObject(); + json.name("version").value(metadata.getVersion()); + json.name("num_rows").value(metadata.getNum_rows()); + json.name("created_by").beginObject(); + json.name("present").value(metadata.isSetCreated_by()); + json.name("value"); + if (metadata.isSetCreated_by()) { + json.value(metadata.getCreated_by()); + } else { + json.nullValue(); + } + json.endObject(); + json.name("schema_leaf_count").value(leaves.size()); + appendSchemaLeaves(json, leaves); + appendColumnOrders(json, footer.rawColumnOrders, leaves); + List rowGroups = metadata.getRow_groups(); + json.name("row_group_count").value(rowGroups == null ? 0 : rowGroups.size()); + json.name("row_groups").beginArray(); + if (rowGroups != null) { + for (int rowGroupOrdinal = 0; rowGroupOrdinal < rowGroups.size(); rowGroupOrdinal++) { + appendRowGroup(json, rowGroups.get(rowGroupOrdinal), rowGroupOrdinal, leaves); + } + } + json.endArray(); + json.endObject(); + } + + private static void appendSchemaLeaves(JsonWriter json, List leaves) { + json.name("schema_leaves").beginArray(); + for (Leaf leaf : leaves) { + SchemaElement schema = leaf.schema; + json.beginObject(); + json.name("ordinal").value(leaf.ordinal); + appendPath(json, "path", leaf.path); + json.name("physical_type").value(schema.getType().name()); + appendOptionalInteger(json, "type_length", schema.isSetType_length(), schema.getType_length()); + appendOptionalEnum(json, "converted_type", schema.isSetConverted_type(), + schema.isSetConverted_type() ? schema.getConverted_type().name() : null); + appendOptionalInteger(json, "scale", schema.isSetScale(), schema.getScale()); + appendOptionalInteger(json, "precision", schema.isSetPrecision(), schema.getPrecision()); + appendOptionalEnum(json, "repetition_type", schema.isSetRepetition_type(), + schema.isSetRepetition_type() ? schema.getRepetition_type().name() : null); + appendOptionalInteger(json, "field_id", schema.isSetField_id(), schema.getField_id()); + appendLogicalTypeDescriptor(json, schema); + json.endObject(); + } + json.endArray(); + } + + private static void appendLogicalTypeDescriptor(JsonWriter json, SchemaElement schema) { + LogicalType logical = schema.isSetLogicalType() ? schema.getLogicalType() : null; + LogicalType._Fields member = logical == null ? null : logical.getSetField(); + json.name("logical_type").beginObject(); + json.name("present").value(logical != null); + json.name("member").value(member == null ? null : member.getFieldName()); + json.name("parameters"); + if (member == null || !hasLogicalParameters(member)) { + json.nullValue(); + } else { + json.beginObject(); + appendLogicalParameters(json, logical, member); + json.endObject(); + } + json.endObject(); + } + + private static boolean hasLogicalParameters(LogicalType._Fields member) { + return member == LogicalType._Fields.INTEGER + || member == LogicalType._Fields.DECIMAL + || member == LogicalType._Fields.TIME + || member == LogicalType._Fields.TIMESTAMP + || member == LogicalType._Fields.VARIANT + || member == LogicalType._Fields.GEOMETRY + || member == LogicalType._Fields.GEOGRAPHY; + } + + private static void appendLogicalParameters( + JsonWriter json, + LogicalType logical, + LogicalType._Fields member + ) { + if (member == LogicalType._Fields.INTEGER) { + IntType integer = logical.getINTEGER(); + json.name("bit_width").value(integer.getBitWidth()); + json.name("is_signed").value(integer.isIsSigned()); + } else if (member == LogicalType._Fields.DECIMAL) { + DecimalType decimal = logical.getDECIMAL(); + json.name("scale").value(decimal.getScale()); + json.name("precision").value(decimal.getPrecision()); + } else if (member == LogicalType._Fields.TIME) { + TimeType time = logical.getTIME(); + json.name("unit").value(timeUnitName(time.getUnit())); + json.name("is_adjusted_to_utc").value(time.isIsAdjustedToUTC()); + } else if (member == LogicalType._Fields.TIMESTAMP) { + TimestampType timestamp = logical.getTIMESTAMP(); + json.name("unit").value(timeUnitName(timestamp.getUnit())); + json.name("is_adjusted_to_utc").value(timestamp.isIsAdjustedToUTC()); + } else if (member == LogicalType._Fields.VARIANT) { + VariantType variant = logical.getVARIANT(); + appendOptionalInteger(json, "specification_version", + variant.isSetSpecification_version(), variant.getSpecification_version()); + } else if (member == LogicalType._Fields.GEOMETRY) { + GeometryType geometry = logical.getGEOMETRY(); + appendOptionalString(json, "crs", geometry.isSetCrs(), geometry.getCrs()); + } else { + GeographyType geography = logical.getGEOGRAPHY(); + appendOptionalString(json, "crs", geography.isSetCrs(), geography.getCrs()); + appendOptionalEnum(json, "algorithm", geography.isSetAlgorithm(), + geography.isSetAlgorithm() ? geography.getAlgorithm().name() : null); + } + } + + private static String timeUnitName(TimeUnit unit) { + return unit == null || unit.getSetField() == null ? null : unit.getSetField().getFieldName(); + } + + private static void appendOptionalInteger( + JsonWriter json, + String name, + boolean present, + long value + ) { + json.name(name).beginObject(); + json.name("present").value(present); + json.name("value"); + if (present) { + json.value(value); + } else { + json.nullValue(); + } + json.endObject(); + } + + private static void appendOptionalString( + JsonWriter json, + String name, + boolean present, + String value + ) { + json.name(name).beginObject(); + json.name("present").value(present); + json.name("value"); + if (present) { + json.value(value); + } else { + json.nullValue(); + } + json.endObject(); + } + + private static void appendOptionalEnum( + JsonWriter json, + String name, + boolean present, + String value + ) { + appendOptionalString(json, name, present, value); + } + + private static void appendColumnOrders( + JsonWriter json, + RawColumnOrders orders, + List leaves + ) { + json.name("column_orders").beginObject(); + json.name("present").value(orders.present); + json.name("count"); + if (!orders.present) { + json.nullValue(); + } else { + json.value(orders.values.size()); + } + json.name("values").beginArray(); + if (orders.present) { + for (int ordinal = 0; ordinal < orders.values.size(); ordinal++) { + RawColumnOrder order = orders.values.get(ordinal); + Leaf leaf = ordinal < leaves.size() ? leaves.get(ordinal) : null; + json.beginObject(); + json.name("ordinal").value(ordinal); + json.name("schema_leaf_ordinal"); + if (leaf == null) { + json.nullValue(); + } else { + json.value(leaf.ordinal); + } + json.name("state").value(order.state); + json.name("field_id"); + if (order.fieldId == null) { + json.nullValue(); + } else { + json.value(order.fieldId); + } + json.name("wire_type"); + if (order.wireType == null) { + json.nullValue(); + } else { + json.value(order.wireType); + } + json.name("header_hex").value(order.headerHex); + json.name("member").value(order.member); + appendPath(json, "path", leaf == null ? null : leaf.path); + json.name("physical_type").value( + leaf == null || !leaf.schema.isSetType() ? null : leaf.schema.getType().name()); + json.name("logical_type").value(logicalTypeName(leaf == null ? null : leaf.schema)); + json.endObject(); + } + } + json.endArray(); + json.endObject(); + } + + private static void appendRowGroup( + JsonWriter json, + RowGroup rowGroup, + int ordinal, + List leaves + ) { + json.beginObject(); + json.name("ordinal").value(ordinal); + json.name("num_rows").value(rowGroup.getNum_rows()); + List columns = rowGroup.getColumns(); + json.name("column_count").value(columns == null ? 0 : columns.size()); + json.name("columns").beginArray(); + if (columns != null) { + for (int columnOrdinal = 0; columnOrdinal < columns.size(); columnOrdinal++) { + appendColumn(json, columns.get(columnOrdinal), columnOrdinal, leaves.get(columnOrdinal)); + } + } + json.endArray(); + json.endObject(); + } + + private static void appendColumn( + JsonWriter json, + ColumnChunk chunk, + int ordinal, + Leaf leaf + ) { + ColumnMetaData metadata = chunk == null ? null : chunk.getMeta_data(); + List path = metadata == null ? null : metadata.getPath_in_schema(); + Type physicalType = metadata == null ? null : metadata.getType(); + boolean float16 = leaf != null && "FLOAT16".equals(logicalTypeName(leaf.schema)); + json.beginObject(); + json.name("ordinal").value(ordinal); + json.name("schema_leaf_ordinal"); + if (leaf == null) { + json.nullValue(); + } else { + json.value(leaf.ordinal); + } + json.name("metadata_present").value(metadata != null); + appendPath(json, "path", path); + json.name("physical_type").value(physicalType == null ? null : physicalType.name()); + json.name("logical_type").value(logicalTypeName(leaf == null ? null : leaf.schema)); + json.name("num_values"); + if (metadata == null) { + json.nullValue(); + } else { + json.value(metadata.getNum_values()); + } + appendStatistics(json, metadata == null ? null : metadata.getStatistics(), physicalType, float16); + json.endObject(); + } + + private static void appendStatistics(JsonWriter json, Statistics statistics, Type physicalType, boolean float16) { + json.name("statistics").beginObject(); + json.name("present").value(statistics != null); + json.name("fields").beginArray(); + for (int fieldId = 1; fieldId <= 9; fieldId++) { + appendStatisticsField(json, statistics, fieldId, physicalType, float16); + } + json.endArray(); + json.endObject(); + } + + private static void appendStatisticsField( + JsonWriter json, + Statistics statistics, + int fieldId, + Type physicalType, + boolean float16 + ) { + Statistics._Fields field = Statistics._Fields.findByThriftId(fieldId); + boolean present = statistics != null && statistics.isSet(field); + json.beginObject(); + json.name("field_id").value(fieldId); + json.name("name").value(field.getFieldName()); + json.name("present").value(present); + json.name("value"); + if (!present) { + json.nullValue(); + } else if (fieldId == 1 || fieldId == 2 || fieldId == 5 || fieldId == 6) { + appendBinary(json, binaryField(statistics, fieldId), physicalType, float16); + } else if (fieldId == 7) { + json.value(statistics.isIs_max_value_exact()); + } else if (fieldId == 8) { + json.value(statistics.isIs_min_value_exact()); + } else if (fieldId == 3) { + json.value(statistics.getNull_count()); + } else if (fieldId == 4) { + json.value(statistics.getDistinct_count()); + } else { + json.value(statistics.getNan_count()); + } + json.endObject(); + } + + private static ByteBuffer binaryField(Statistics statistics, int fieldId) { + ByteBuffer buffer; + if (fieldId == 1) { + buffer = statistics.bufferForMax(); + } else if (fieldId == 2) { + buffer = statistics.bufferForMin(); + } else if (fieldId == 5) { + buffer = statistics.bufferForMax_value(); + } else { + buffer = statistics.bufferForMin_value(); + } + return buffer.duplicate(); + } + + private static void appendBinary(JsonWriter json, ByteBuffer bytes, Type physicalType, boolean float16) { + json.beginObject(); + json.name("hex").hexValue(bytes, false, false); + json.name("byte_length").value(bytes.remaining()); + int width = floatingWidth(physicalType, float16); + json.name("float_bits"); + if (width == 0) { + json.nullValue(); + } else { + json.beginObject(); + json.name("width").value(width); + boolean valid = bytes.remaining() * 8 == width; + json.name("valid_width").value(valid); + json.name("hex"); + if (valid) { + json.hexValue(bytes, true, true); + } else { + json.nullValue(); + } + json.endObject(); + } + json.endObject(); + } + + private static int floatingWidth(Type physicalType, boolean float16) { + if (float16) { + return 16; + } + if (physicalType == Type.FLOAT) { + return 32; + } + if (physicalType == Type.DOUBLE) { + return 64; + } + return 0; + } + + private static void appendPath(JsonWriter json, String name, List path) { + json.name(name); + if (path == null) { + json.nullValue(); + return; + } + json.beginArray(); + for (String component : path) { + json.value(component); + } + json.endArray(); + } + + private static String logicalTypeName(SchemaElement schema) { + if (schema == null || !schema.isSetLogicalType() || schema.getLogicalType().getSetField() == null) { + return null; + } + return schema.getLogicalType().getSetField().getFieldName(); + } + + private static RawColumnOrders readRawColumnOrders(byte[] footerBytes) + throws IOException, TException { + TMemoryInputTransport transport = new TMemoryInputTransport(footerBytes); + TCompactProtocol protocol = new TCompactProtocol( + transport, MAX_THRIFT_BINARY_BYTES, MAX_THRIFT_CONTAINER_ELEMENTS); + boolean present = false; + List values = List.of(); + protocol.readStructBegin(); + while (true) { + TField field = protocol.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + if (field.id == 7) { + if (present) { + throw new IOException("FileMetaData contains duplicate column_orders fields"); + } + present = true; + if (field.type != TType.LIST) { + throw new IOException("FileMetaData column_orders has the wrong wire type"); + } + TList list = protocol.readListBegin(); + if (list.elemType != TType.STRUCT || list.size < 0 || list.size > MAX_SCHEMA_ELEMENTS) { + throw new IOException("FileMetaData column_orders has an invalid list header"); + } + List captured = new ArrayList<>(list.size); + for (int ordinal = 0; ordinal < list.size; ordinal++) { + protocol.readStructBegin(); + int headerOffset = transport.getBufferPosition(); + TField member = protocol.readFieldBegin(); + String headerHex = hex(Arrays.copyOfRange( + footerBytes, headerOffset, transport.getBufferPosition())); + if (member.type == TType.STOP) { + captured.add(new RawColumnOrder( + null, null, headerHex, "empty", null, headerOffset)); + } else { + int fieldId = member.id; + byte wireType = member.type; + String state = rawColumnOrderState(fieldId, wireType); + String name = "known".equals(state) ? + (fieldId == 1 ? "TYPE_ORDER" : "IEEE_754_TOTAL_ORDER") : null; + TProtocolUtil.skip(protocol, wireType, MAX_THRIFT_DEPTH); + protocol.readFieldEnd(); + TField extra = protocol.readFieldBegin(); + if (extra.type != TType.STOP) { + throw new IOException("ColumnOrder union has more than one raw member"); + } + captured.add(new RawColumnOrder( + fieldId, wireType, headerHex, state, name, headerOffset)); + } + protocol.readStructEnd(); + } + protocol.readListEnd(); + values = Collections.unmodifiableList(captured); + } else { + TProtocolUtil.skip(protocol, field.type, MAX_THRIFT_DEPTH); + } + protocol.readFieldEnd(); + } + protocol.readStructEnd(); + if (transport.getBytesRemainingInBuffer() != 0) { + throw new IOException("raw Compact-Thrift footer scan left trailing bytes"); + } + return new RawColumnOrders(present, values); + } + + private static byte[] withoutColumnOrders(byte[] footerBytes) throws IOException, TException { + TMemoryInputTransport inputTransport = new TMemoryInputTransport(footerBytes); + TCompactProtocol input = new TCompactProtocol( + inputTransport, MAX_THRIFT_BINARY_BYTES, MAX_THRIFT_CONTAINER_ELEMENTS); + ByteArrayOutputStream encoded = new ByteArrayOutputStream(footerBytes.length); + TIOStreamTransport outputTransport = new TIOStreamTransport(encoded); + TCompactProtocol output = new TCompactProtocol(outputTransport); + input.readStructBegin(); + output.writeStructBegin(new TStruct("FileMetaData")); + while (true) { + TField field = input.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + if (field.id == 7) { + TProtocolUtil.skip(input, field.type, MAX_THRIFT_DEPTH); + } else { + output.writeFieldBegin(new TField(field.name, field.type, field.id)); + copyValue(input, output, field.type, 0); + output.writeFieldEnd(); + } + input.readFieldEnd(); + } + input.readStructEnd(); + output.writeFieldStop(); + output.writeStructEnd(); + outputTransport.flush(); + if (inputTransport.getBytesRemainingInBuffer() != 0) { + throw new IOException("Compact-Thrift footer transcode left trailing bytes"); + } + return encoded.toByteArray(); + } + + private static void copyValue( + TCompactProtocol input, + TCompactProtocol output, + byte type, + int depth + ) throws TException { + if (depth > MAX_THRIFT_DEPTH) { + throw new TException("raw Compact-Thrift nesting exceeds the scanner limit"); + } + switch (type) { + case TType.BOOL: + output.writeBool(input.readBool()); + return; + case TType.BYTE: + output.writeByte(input.readByte()); + return; + case TType.I16: + output.writeI16(input.readI16()); + return; + case TType.I32: + case TType.ENUM: + output.writeI32(input.readI32()); + return; + case TType.I64: + output.writeI64(input.readI64()); + return; + case TType.DOUBLE: + output.writeDouble(input.readDouble()); + return; + case TType.STRING: + output.writeBinary(input.readBinary()); + return; + case TType.UUID: + output.writeUuid(input.readUuid()); + return; + case TType.STRUCT: + copyStruct(input, output, depth + 1); + return; + case TType.MAP: + TMap map = input.readMapBegin(); + output.writeMapBegin(new TMap(map.keyType, map.valueType, map.size)); + for (int index = 0; index < map.size; index++) { + copyValue(input, output, map.keyType, depth + 1); + copyValue(input, output, map.valueType, depth + 1); + } + input.readMapEnd(); + output.writeMapEnd(); + return; + case TType.LIST: + TList list = input.readListBegin(); + output.writeListBegin(new TList(list.elemType, list.size)); + for (int index = 0; index < list.size; index++) { + copyValue(input, output, list.elemType, depth + 1); + } + input.readListEnd(); + output.writeListEnd(); + return; + case TType.SET: + TSet set = input.readSetBegin(); + output.writeSetBegin(new TSet(set.elemType, set.size)); + for (int index = 0; index < set.size; index++) { + copyValue(input, output, set.elemType, depth + 1); + } + input.readSetEnd(); + output.writeSetEnd(); + return; + default: + throw new TException("unsupported raw Compact-Thrift type: " + type); + } + } + + private static void copyStruct(TCompactProtocol input, TCompactProtocol output, int depth) + throws TException { + input.readStructBegin(); + output.writeStructBegin(new TStruct()); + while (true) { + TField field = input.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + output.writeFieldBegin(new TField(field.name, field.type, field.id)); + copyValue(input, output, field.type, depth); + input.readFieldEnd(); + output.writeFieldEnd(); + } + input.readStructEnd(); + output.writeFieldStop(); + output.writeStructEnd(); + } + + private static String rawColumnOrderState(int fieldId, byte wireType) { + if (fieldId == 1 || fieldId == 2) { + return wireType == TType.STRUCT ? "known" : "wrong_type"; + } + return "unknown"; + } + + private static List schemaLeaves(FileMetaData metadata) throws IOException { + if (!metadata.isSetVersion() || metadata.getVersion() <= 0) { + throw new IOException("FileMetaData version must be present and positive"); + } + if (!metadata.isSetNum_rows() || metadata.getNum_rows() < 0) { + throw new IOException("FileMetaData num_rows must be present and nonnegative"); + } + if (!metadata.isSetRow_groups() || metadata.getRow_groups() == null) { + throw new IOException("FileMetaData row_groups must be present"); + } + List schema = metadata.getSchema(); + List leaves = new ArrayList<>(); + if (!metadata.isSetSchema() || schema == null || schema.isEmpty()) { + throw new IOException("FileMetaData schema must be present and nonempty"); + } + if (schema.size() > MAX_SCHEMA_ELEMENTS) { + throw new IOException("FileMetaData schema exceeds the scanner element limit"); + } + SchemaElement root = schema.get(0); + if (root == null || root.getName() == null || root.getName().isEmpty() || root.isSetType() + || !root.isSetNum_children() || root.getNum_children() < 0) { + throw new IOException("FileMetaData schema root is invalid"); + } + int rootChildren = root.getNum_children(); + Deque stack = new ArrayDeque<>(); + stack.push(new SchemaFrame(rootChildren, List.of())); + Set> paths = new HashSet<>(); + for (int index = 1; index < schema.size(); index++) { + while (!stack.isEmpty() && stack.peek().remaining == 0) { + stack.pop(); + } + if (stack.isEmpty()) { + throw new IOException("FileMetaData schema has an orphan element"); + } + SchemaElement element = schema.get(index); + if (element == null || element.getName() == null || element.getName().isEmpty()) { + throw new IOException("FileMetaData schema has an unnamed element"); + } + List parent = stack.peek().path; + stack.peek().remaining--; + List path = new ArrayList<>(parent.size() + 1); + path.addAll(parent); + path.add(element.getName()); + List stablePath = Collections.unmodifiableList(new ArrayList<>(path)); + if (!paths.add(stablePath)) { + throw new IOException("FileMetaData schema has a duplicate path: " + stablePath); + } + if (element.isSetType()) { + if (element.isSetNum_children() && element.getNum_children() != 0) { + throw new IOException("FileMetaData leaf has child elements: " + stablePath); + } + leaves.add(new Leaf(leaves.size(), stablePath, element)); + } else { + if (!element.isSetNum_children() || element.getNum_children() < 0) { + throw new IOException("FileMetaData group has an invalid child count: " + stablePath); + } + stack.push(new SchemaFrame(element.getNum_children(), stablePath)); + } + } + while (!stack.isEmpty() && stack.peek().remaining == 0) { + stack.pop(); + } + if (!stack.isEmpty()) { + throw new IOException("FileMetaData schema ends before all declared children"); + } + return leaves; + } + + static void validateMetadata(FileMetaData metadata) throws IOException { + List leaves = schemaLeaves(metadata); + Map, Leaf> leavesByPath = new HashMap<>(); + for (Leaf leaf : leaves) { + leavesByPath.put(leaf.path, leaf); + } + validateRowGroups(metadata, leaves, leavesByPath); + } + + private static void validateRowGroups( + FileMetaData metadata, + List leaves, + Map, Leaf> leavesByPath + ) throws IOException { + List rowGroups = metadata.getRow_groups(); + for (int rowGroupOrdinal = 0; rowGroupOrdinal < rowGroups.size(); rowGroupOrdinal++) { + RowGroup rowGroup = rowGroups.get(rowGroupOrdinal); + if (rowGroup == null || !rowGroup.isSetColumns() || rowGroup.getColumns() == null) { + throw new IOException("RowGroup columns must be present: " + rowGroupOrdinal); + } + if (!rowGroup.isSetTotal_byte_size() || rowGroup.getTotal_byte_size() < 0) { + throw new IOException( + "RowGroup total_byte_size must be present and nonnegative: " + rowGroupOrdinal); + } + if (!rowGroup.isSetNum_rows() || rowGroup.getNum_rows() < 0) { + throw new IOException("RowGroup num_rows must be present and nonnegative: " + rowGroupOrdinal); + } + if (rowGroup.isSetFile_offset() && rowGroup.getFile_offset() < 0) { + throw new IOException("RowGroup file_offset must be nonnegative: " + rowGroupOrdinal); + } + if (rowGroup.isSetTotal_compressed_size() && rowGroup.getTotal_compressed_size() < 0) { + throw new IOException( + "RowGroup total_compressed_size must be nonnegative: " + rowGroupOrdinal); + } + List columns = rowGroup.getColumns(); + if (columns.size() != leaves.size()) { + throw new IOException( + "RowGroup column count differs from the schema leaf count: " + rowGroupOrdinal); + } + Set> columnPaths = new HashSet<>(); + for (int columnOrdinal = 0; columnOrdinal < columns.size(); columnOrdinal++) { + validateColumnChunk( + columns.get(columnOrdinal), rowGroupOrdinal, columnOrdinal, leaves.get(columnOrdinal), + leavesByPath, columnPaths); + } + } + } + + private static void validateColumnChunk( + ColumnChunk chunk, + int rowGroupOrdinal, + int columnOrdinal, + Leaf expectedLeaf, + Map, Leaf> leavesByPath, + Set> columnPaths + ) throws IOException { + String location = rowGroupOrdinal + "/" + columnOrdinal; + if (chunk == null || !chunk.isSetFile_offset() || chunk.getFile_offset() < 0) { + throw new IOException("ColumnChunk file_offset must be present and nonnegative: " + location); + } + validateOptionalOffset(chunk.isSetOffset_index_offset(), chunk.getOffset_index_offset(), + "offset_index_offset", location); + validateOptionalLength(chunk.isSetOffset_index_length(), chunk.getOffset_index_length(), + "offset_index_length", location); + validateOptionalOffset(chunk.isSetColumn_index_offset(), chunk.getColumn_index_offset(), + "column_index_offset", location); + validateOptionalLength(chunk.isSetColumn_index_length(), chunk.getColumn_index_length(), + "column_index_length", location); + ColumnMetaData metadata = chunk.getMeta_data(); + if (metadata == null) { + return; + } + if (!metadata.isSetType() || !metadata.isSetEncodings() || metadata.getEncodings() == null + || !metadata.isSetPath_in_schema() || metadata.getPath_in_schema() == null + || metadata.getPath_in_schema().isEmpty() || !metadata.isSetCodec()) { + throw new IOException("ColumnMetaData required descriptors are missing: " + location); + } + List path = metadata.getPath_in_schema(); + for (String component : path) { + if (component == null || component.isEmpty()) { + throw new IOException("ColumnMetaData path contains an empty component: " + location); + } + } + Leaf leaf = leavesByPath.get(path); + if (leaf == null) { + throw new IOException("ColumnMetaData path is not a schema leaf: " + path); + } + if (metadata.getType() != leaf.schema.getType()) { + throw new IOException("ColumnMetaData physical type differs from schema leaf: " + path); + } + if (leaf != expectedLeaf) { + throw new IOException("ColumnMetaData order differs from the schema leaf order: " + path); + } + if (!columnPaths.add(path)) { + throw new IOException("RowGroup contains a duplicate column path: " + path); + } + if (!metadata.isSetNum_values() || metadata.getNum_values() < 0) { + throw new IOException("ColumnMetaData num_values must be present and nonnegative: " + location); + } + if (!metadata.isSetTotal_uncompressed_size() || metadata.getTotal_uncompressed_size() < 0) { + throw new IOException( + "ColumnMetaData total_uncompressed_size must be present and nonnegative: " + location); + } + if (!metadata.isSetTotal_compressed_size() || metadata.getTotal_compressed_size() < 0) { + throw new IOException( + "ColumnMetaData total_compressed_size must be present and nonnegative: " + location); + } + if (!metadata.isSetData_page_offset() || metadata.getData_page_offset() < 0) { + throw new IOException( + "ColumnMetaData data_page_offset must be present and nonnegative: " + location); + } + validateOptionalOffset(metadata.isSetIndex_page_offset(), metadata.getIndex_page_offset(), + "index_page_offset", location); + validateOptionalOffset(metadata.isSetDictionary_page_offset(), metadata.getDictionary_page_offset(), + "dictionary_page_offset", location); + validateOptionalOffset(metadata.isSetBloom_filter_offset(), metadata.getBloom_filter_offset(), + "bloom_filter_offset", location); + validateOptionalLength(metadata.isSetBloom_filter_length(), metadata.getBloom_filter_length(), + "bloom_filter_length", location); + } + + private static void validateOptionalOffset( + boolean present, + long value, + String name, + String location + ) throws IOException { + if (present && value < 0) { + throw new IOException(name + " must be nonnegative: " + location); + } + } + + private static void validateOptionalLength( + boolean present, + int value, + String name, + String location + ) throws IOException { + if (present && value < 0) { + throw new IOException(name + " must be nonnegative: " + location); + } + } + + private static List collectInputs(List inputs) throws IOException { + List files = new ArrayList<>(); + for (Path input : inputs) { + Path normalized = input.toAbsolutePath().normalize(); + if (Files.isRegularFile(normalized)) { + files.add(new InputFile(normalized.getFileName().toString(), normalized)); + } else if (Files.isDirectory(normalized)) { + try (Stream stream = Files.walk(normalized)) { + java.util.Iterator paths = stream.filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".parquet")) + .iterator(); + while (paths.hasNext()) { + Path path = paths.next(); + files.add(new InputFile( + normalized.relativize(path).toString().replace(path.getFileSystem().getSeparator(), "/"), + path)); + requireInputCount(files.size()); + } + } + } else { + throw new IOException("input does not exist or is not a regular file or directory: " + input); + } + requireInputCount(files.size()); + } + files.sort(Comparator.comparing(file -> file.label)); + Set labels = new HashSet<>(); + for (InputFile file : files) { + if (!labels.add(file.label)) { + throw new IOException("duplicate deterministic input label: " + file.label); + } + } + if (files.isEmpty()) { + throw new IOException("no Parquet input files found"); + } + return files; + } + + private static void requireInputCount(int count) throws IOException { + if (count > MAX_INPUT_FILES) { + throw new IOException("Parquet input count exceeds the scanner limit: " + count); + } + } + + private static void validateOutput(List inputs, Path output) throws IOException { + if (output == null) { + return; + } + Path absolute = output.toAbsolutePath().normalize(); + for (InputFile input : inputs) { + if (absolute.equals(input.path)) { + throw new IOException("output path aliases an input path: " + output); + } + if (Files.exists(absolute) && Files.isSameFile(absolute, input.path)) { + throw new IOException("output file aliases an input file: " + output); + } + } + } + + private static void writeEvidence(List evidence, Path output) throws IOException { + if (output == null) { + for (String record : evidence) { + System.out.print(record); + System.out.print('\n'); + } + return; + } + Path absolute = output.toAbsolutePath().normalize(); + Path parent = absolute.getParent(); + if (parent == null || !Files.isDirectory(parent)) { + throw new IOException("output parent directory does not exist: " + output); + } + Path temporary = Files.createTempFile(parent, ".raw-footer-", ".tmp"); + boolean moved = false; + try { + try (BufferedWriter writer = Files.newBufferedWriter(temporary, StandardCharsets.UTF_8)) { + for (String record : evidence) { + writer.write(record); + writer.write('\n'); + } + } + Files.move(temporary, absolute, + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + moved = true; + } finally { + if (!moved) { + Files.deleteIfExists(temporary); + } + } + } + + private static void requireArgument(String[] arguments, int index, String option) { + if (index >= arguments.length) { + throw new IllegalArgumentException(option + " requires a value"); + } + } + + private static void requireMagic(byte[] actual, String position, Path path) throws IOException { + if (!Arrays.equals(actual, MAGIC)) { + throw new IOException("invalid " + position + " Parquet magic: " + path); + } + } + + private static long unsignedLittleEndianInt(byte[] bytes) { + return (bytes[0] & 0xffL) + | ((bytes[1] & 0xffL) << 8) + | ((bytes[2] & 0xffL) << 16) + | ((bytes[3] & 0xffL) << 24); + } + + private static String sha256(RandomAccessFile file, long expectedSize) throws IOException { + MessageDigest digest = sha256Digest(); + byte[] buffer = new byte[16 * 1024]; + long remaining = expectedSize; + file.seek(0); + while (remaining > 0) { + int count = file.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (count < 0) { + throw new IOException("Parquet input became shorter during hashing"); + } + digest.update(buffer, 0, count); + remaining -= count; + } + return hex(digest.digest()); + } + + private static boolean sameFileSnapshot(BasicFileAttributes before, BasicFileAttributes after) { + if (!before.isRegularFile() || !after.isRegularFile() || before.size() != after.size() + || !before.lastModifiedTime().equals(after.lastModifiedTime())) { + return false; + } + Object beforeKey = before.fileKey(); + Object afterKey = after.fileKey(); + return beforeKey == null || afterKey == null || beforeKey.equals(afterKey); + } + + private static String sha256(byte[] bytes) { + MessageDigest digest = sha256Digest(); + return hex(digest.digest(bytes)); + } + + private static MessageDigest sha256Digest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("Java runtime does not provide SHA-256", exception); + } + } + + private static String hex(byte[] bytes) { + StringBuilder result = new StringBuilder(bytes.length * 2); + for (byte value : bytes) { + int unsigned = value & 0xff; + result.append("0123456789abcdef".charAt(unsigned >>> 4)); + result.append("0123456789abcdef".charAt(unsigned & 0x0f)); + } + return result.toString(); + } + +} diff --git a/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestFixture.java b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestFixture.java new file mode 100644 index 0000000..538ca58 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestFixture.java @@ -0,0 +1,342 @@ +package org.julialang.parquet.n6.raw; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.apache.parquet.format.ColumnChunk; +import org.apache.parquet.format.ColumnMetaData; +import org.apache.parquet.format.ColumnOrder; +import org.apache.parquet.format.CompressionCodec; +import org.apache.parquet.format.ConvertedType; +import org.apache.parquet.format.DecimalType; +import org.apache.parquet.format.Encoding; +import org.apache.parquet.format.FieldRepetitionType; +import org.apache.parquet.format.FileMetaData; +import org.apache.parquet.format.Float16Type; +import org.apache.parquet.format.IEEE754TotalOrder; +import org.apache.parquet.format.IntType; +import org.apache.parquet.format.LogicalType; +import org.apache.parquet.format.MicroSeconds; +import org.apache.parquet.format.NanoSeconds; +import org.apache.parquet.format.RowGroup; +import org.apache.parquet.format.SchemaElement; +import org.apache.parquet.format.Statistics; +import org.apache.parquet.format.TimeType; +import org.apache.parquet.format.TimeUnit; +import org.apache.parquet.format.TimestampType; +import org.apache.parquet.format.Type; +import org.apache.parquet.format.TypeDefinedOrder; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.protocol.TType; +import org.apache.thrift.transport.TIOStreamTransport; + +public final class SelfTestFixture { + private static final byte[] MAGIC = new byte[] {'P', 'A', 'R', '1'}; + + private SelfTestFixture() { + } + + public static void main(String[] arguments) throws Exception { + if (arguments.length != 1) { + throw new IllegalArgumentException("usage: SelfTestFixture "); + } + write(Path.of(arguments[0])); + } + + static void write(Path output) throws IOException, TException { + write(output, metadata()); + } + + static void write(Path output, FileMetaData metadata) throws IOException, TException { + ByteArrayOutputStream encoded = new ByteArrayOutputStream(); + TIOStreamTransport transport = new TIOStreamTransport(encoded); + metadata.write(new TCompactProtocol(transport)); + transport.flush(); + writeRawFooter(output, encoded.toByteArray()); + } + + static void writeMissingRequired( + Path output, + boolean omitRowGroupNumRows, + boolean omitColumnNumValues + ) throws IOException, TException { + FileMetaData metadata = metadata(); + ByteArrayOutputStream encoded = new ByteArrayOutputStream(); + TIOStreamTransport transport = new TIOStreamTransport(encoded); + TCompactProtocol protocol = new TCompactProtocol(transport); + protocol.writeStructBegin(new TStruct("FileMetaData")); + writeField(protocol, TType.I32, 1, () -> protocol.writeI32(metadata.getVersion())); + writeField(protocol, TType.LIST, 2, () -> { + protocol.writeListBegin(new TList(TType.STRUCT, metadata.getSchemaSize())); + for (SchemaElement element : metadata.getSchema()) { + element.write(protocol); + } + protocol.writeListEnd(); + }); + writeField(protocol, TType.I64, 3, () -> protocol.writeI64(metadata.getNum_rows())); + writeField(protocol, TType.LIST, 4, () -> { + protocol.writeListBegin(new TList(TType.STRUCT, metadata.getRow_groupsSize())); + for (RowGroup rowGroup : metadata.getRow_groups()) { + writeRowGroup(protocol, rowGroup, omitRowGroupNumRows, omitColumnNumValues); + } + protocol.writeListEnd(); + }); + if (metadata.isSetCreated_by()) { + writeField(protocol, TType.STRING, 6, + () -> protocol.writeString(metadata.getCreated_by())); + } + writeField(protocol, TType.LIST, 7, () -> { + protocol.writeListBegin(new TList(TType.STRUCT, metadata.getColumn_ordersSize())); + for (ColumnOrder order : metadata.getColumn_orders()) { + order.write(protocol); + } + protocol.writeListEnd(); + }); + protocol.writeFieldStop(); + protocol.writeStructEnd(); + transport.flush(); + writeRawFooter(output, encoded.toByteArray()); + } + + static void writeRawFooter(Path output, byte[] footer) throws IOException { + if (footer.length > Integer.MAX_VALUE - 12) { + throw new IOException("self-test footer is unexpectedly large"); + } + ByteArrayOutputStream file = new ByteArrayOutputStream(footer.length + 12); + file.write(MAGIC); + file.write(footer); + writeLittleEndianInt(file, footer.length); + file.write(MAGIC); + Path absolute = output.toAbsolutePath().normalize(); + Path parent = absolute.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + Files.write(absolute, file.toByteArray()); + } + + @FunctionalInterface + private interface ThriftAction { + void run() throws TException; + } + + private static void writeField( + TCompactProtocol protocol, + byte type, + int id, + ThriftAction action + ) throws TException { + protocol.writeFieldBegin(new TField("", type, (short) id)); + action.run(); + protocol.writeFieldEnd(); + } + + private static void writeRowGroup( + TCompactProtocol protocol, + RowGroup rowGroup, + boolean omitNumRows, + boolean omitColumnNumValues + ) throws TException { + protocol.writeStructBegin(new TStruct("RowGroup")); + writeField(protocol, TType.LIST, 1, () -> { + protocol.writeListBegin(new TList(TType.STRUCT, rowGroup.getColumnsSize())); + for (ColumnChunk chunk : rowGroup.getColumns()) { + writeColumnChunk(protocol, chunk, omitColumnNumValues); + } + protocol.writeListEnd(); + }); + writeField(protocol, TType.I64, 2, () -> protocol.writeI64(rowGroup.getTotal_byte_size())); + if (!omitNumRows) { + writeField(protocol, TType.I64, 3, () -> protocol.writeI64(rowGroup.getNum_rows())); + } + protocol.writeFieldStop(); + protocol.writeStructEnd(); + } + + private static void writeColumnChunk( + TCompactProtocol protocol, + ColumnChunk chunk, + boolean omitNumValues + ) throws TException { + protocol.writeStructBegin(new TStruct("ColumnChunk")); + writeField(protocol, TType.I64, 2, () -> protocol.writeI64(chunk.getFile_offset())); + writeField(protocol, TType.STRUCT, 3, + () -> writeColumnMetaData(protocol, chunk.getMeta_data(), omitNumValues)); + protocol.writeFieldStop(); + protocol.writeStructEnd(); + } + + private static void writeColumnMetaData( + TCompactProtocol protocol, + ColumnMetaData metadata, + boolean omitNumValues + ) throws TException { + protocol.writeStructBegin(new TStruct("ColumnMetaData")); + writeField(protocol, TType.I32, 1, () -> protocol.writeI32(metadata.getType().getValue())); + writeField(protocol, TType.LIST, 2, () -> { + protocol.writeListBegin(new TList(TType.I32, metadata.getEncodingsSize())); + for (Encoding encoding : metadata.getEncodings()) { + protocol.writeI32(encoding.getValue()); + } + protocol.writeListEnd(); + }); + writeField(protocol, TType.LIST, 3, () -> { + protocol.writeListBegin(new TList(TType.STRING, metadata.getPath_in_schemaSize())); + for (String component : metadata.getPath_in_schema()) { + protocol.writeString(component); + } + protocol.writeListEnd(); + }); + writeField(protocol, TType.I32, 4, () -> protocol.writeI32(metadata.getCodec().getValue())); + if (!omitNumValues) { + writeField(protocol, TType.I64, 5, () -> protocol.writeI64(metadata.getNum_values())); + } + writeField(protocol, TType.I64, 6, + () -> protocol.writeI64(metadata.getTotal_uncompressed_size())); + writeField(protocol, TType.I64, 7, + () -> protocol.writeI64(metadata.getTotal_compressed_size())); + writeField(protocol, TType.I64, 9, () -> protocol.writeI64(metadata.getData_page_offset())); + if (metadata.isSetStatistics()) { + writeField(protocol, TType.STRUCT, 12, () -> metadata.getStatistics().write(protocol)); + } + protocol.writeFieldStop(); + protocol.writeStructEnd(); + } + + static FileMetaData metadata() { + SchemaElement root = new SchemaElement("schema").setNum_children(8); + SchemaElement float32 = leaf("float32", Type.FLOAT); + SchemaElement float64 = leaf("float64", Type.DOUBLE); + SchemaElement float16 = leaf("float16", Type.FIXED_LEN_BYTE_ARRAY) + .setType_length(2) + .setLogicalType(LogicalType.FLOAT16(new Float16Type())); + SchemaElement binary = leaf("binary", Type.BYTE_ARRAY); + SchemaElement integer = leaf("uint16", Type.INT32) + .setType_length(16) + .setConverted_type(ConvertedType.UINT_16) + .setField_id(17) + .setLogicalType(LogicalType.INTEGER(new IntType((byte) 16, false))); + SchemaElement decimal = leaf("decimal4", Type.FIXED_LEN_BYTE_ARRAY) + .setType_length(4) + .setConverted_type(ConvertedType.DECIMAL) + .setScale(2) + .setPrecision(9) + .setLogicalType(LogicalType.DECIMAL(new DecimalType(2, 9))); + SchemaElement time = leaf("time64", Type.INT64) + .setConverted_type(ConvertedType.TIME_MICROS) + .setLogicalType(LogicalType.TIME(new TimeType( + true, TimeUnit.MICROS(new MicroSeconds())))); + SchemaElement timestamp = leaf("timestamp64", Type.INT64) + .setLogicalType(LogicalType.TIMESTAMP(new TimestampType( + false, TimeUnit.NANOS(new NanoSeconds())))); + List schema = List.of( + root, float32, float64, float16, binary, integer, decimal, time, timestamp); + + Statistics float32Statistics = new Statistics() + .setMax(hex("0000803f")) + .setMin(hex("000080bf")) + .setNull_count(1) + .setDistinct_count(2) + .setMax_value(hex("00000000")) + .setMin_value(hex("00000080")) + .setIs_max_value_exact(true) + .setIs_min_value_exact(false) + .setNan_count(0); + Statistics float64Statistics = new Statistics() + .setNull_count(1) + .setMax_value(hex("420000000000f87f")) + .setMin_value(hex("010000000000f8ff")) + .setIs_max_value_exact(true) + .setIs_min_value_exact(true) + .setNan_count(2); + Statistics float16Statistics = new Statistics() + .setNull_count(0) + .setDistinct_count(2) + .setMax_value(hex("007c")) + .setMin_value(hex("0080")) + .setIs_min_value_exact(true) + .setNan_count(1); + Statistics binaryStatistics = new Statistics() + .setMax(hex("ff")) + .setMin(hex("00")) + .setDistinct_count(3) + .setMax_value(hex("ff007f")) + .setMin_value(hex("00ff")) + .setIs_max_value_exact(false); + + List columns = new ArrayList<>(); + columns.add(column("float32", Type.FLOAT, float32Statistics)); + columns.add(column("float64", Type.DOUBLE, float64Statistics)); + columns.add(column("float16", Type.FIXED_LEN_BYTE_ARRAY, float16Statistics)); + columns.add(column("binary", Type.BYTE_ARRAY, binaryStatistics)); + columns.add(column("uint16", Type.INT32, new Statistics().setNull_count(0))); + columns.add(column("decimal4", Type.FIXED_LEN_BYTE_ARRAY, + new Statistics().setNull_count(0))); + columns.add(column("time64", Type.INT64, new Statistics().setNull_count(0))); + columns.add(column("timestamp64", Type.INT64, new Statistics().setNull_count(0))); + RowGroup rowGroup = new RowGroup(columns, 0, 3); + List orders = List.of( + ColumnOrder.TYPE_ORDER(new TypeDefinedOrder()), + ColumnOrder.IEEE_754_TOTAL_ORDER(new IEEE754TotalOrder()), + ColumnOrder.IEEE_754_TOTAL_ORDER(new IEEE754TotalOrder()), + ColumnOrder.TYPE_ORDER(new TypeDefinedOrder()), + ColumnOrder.TYPE_ORDER(new TypeDefinedOrder()), + ColumnOrder.TYPE_ORDER(new TypeDefinedOrder()), + ColumnOrder.TYPE_ORDER(new TypeDefinedOrder()), + ColumnOrder.TYPE_ORDER(new TypeDefinedOrder())); + return new FileMetaData(1, schema, 3, List.of(rowGroup)) + .setCreated_by("raw-java self-test \u03c0") + .setColumn_orders(orders); + } + + private static SchemaElement leaf(String name, Type type) { + return new SchemaElement(name) + .setType(type) + .setRepetition_type(FieldRepetitionType.OPTIONAL); + } + + private static ColumnChunk column(String name, Type type, Statistics statistics) { + ColumnMetaData metadata = new ColumnMetaData( + type, + List.of(Encoding.PLAIN), + List.of(name), + CompressionCodec.UNCOMPRESSED, + 3, + 0, + 0, + 4) + .setStatistics(statistics); + return new ColumnChunk(4).setMeta_data(metadata); + } + + private static byte[] hex(String value) { + if ((value.length() & 1) != 0) { + throw new IllegalArgumentException("hex value has odd length"); + } + byte[] bytes = new byte[value.length() / 2]; + for (int index = 0; index < bytes.length; index++) { + int high = Character.digit(value.charAt(index * 2), 16); + int low = Character.digit(value.charAt(index * 2 + 1), 16); + if (high < 0 || low < 0) { + throw new IllegalArgumentException("invalid hex value"); + } + bytes[index] = (byte) ((high << 4) | low); + } + return bytes; + } + + private static void writeLittleEndianInt(ByteArrayOutputStream output, int value) { + output.write(value & 0xff); + output.write((value >>> 8) & 0xff); + output.write((value >>> 16) & 0xff); + output.write((value >>> 24) & 0xff); + } +} diff --git a/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestMain.java b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestMain.java new file mode 100644 index 0000000..57165c6 --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/src/org/julialang/parquet/n6/raw/SelfTestMain.java @@ -0,0 +1,651 @@ +package org.julialang.parquet.n6.raw; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import org.apache.parquet.format.FileMetaData; +import org.apache.parquet.format.Statistics; + +public final class SelfTestMain { + @FunctionalInterface + private interface ThrowingAction { + void run() throws Exception; + } + + private SelfTestMain() { + } + + public static void main(String[] arguments) throws Exception { + if (arguments.length != 1) { + throw new IllegalArgumentException("usage: SelfTestMain "); + } + Path fixture = Path.of(arguments[0]).toAbsolutePath().normalize(); + checkGeneratedFieldIds(); + checkDecodedFixture(fixture); + checkEvidence(fixture); + checkRawColumnOrderMutations(fixture); + checkSchemaFailures(); + checkStructuralPreflight(); + checkFailures(fixture); + } + + private static void checkGeneratedFieldIds() { + require(org.apache.parquet.format.ColumnOrder._Fields.TYPE__ORDER.getThriftFieldId() == 1, + "TYPE_ORDER field ID"); + require(org.apache.parquet.format.ColumnOrder._Fields.IEEE_754__TOTAL__ORDER + .getThriftFieldId() == 2, + "IEEE_754_TOTAL_ORDER field ID"); + for (int fieldId = 1; fieldId <= 9; fieldId++) { + Statistics._Fields field = Statistics._Fields.findByThriftId(fieldId); + require(field != null, "Statistics field " + fieldId + " exists"); + require(field.getThriftFieldId() == fieldId, "Statistics field " + fieldId + " identity"); + } + require("max_value".equals(Statistics._Fields.findByThriftId(5).getFieldName()), + "Statistics field 5 name"); + require("min_value".equals(Statistics._Fields.findByThriftId(6).getFieldName()), + "Statistics field 6 name"); + require("is_max_value_exact".equals(Statistics._Fields.findByThriftId(7).getFieldName()), + "Statistics field 7 name"); + require("is_min_value_exact".equals(Statistics._Fields.findByThriftId(8).getFieldName()), + "Statistics field 8 name"); + require("nan_count".equals(Statistics._Fields.findByThriftId(9).getFieldName()), + "Statistics field 9 name"); + } + + private static void checkDecodedFixture(Path fixture) throws Exception { + RawFooterScanner.Footer footer = RawFooterScanner.readFooter(fixture); + FileMetaData metadata = footer.metadata; + require(!metadata.isSetColumn_orders(), "semantic decode excludes raw column orders"); + require(footer.rawColumnOrders.present && footer.rawColumnOrders.values.size() == 8, + "raw column order count"); + require("TYPE_ORDER".equals(footer.rawColumnOrders.values.get(0).member), + "raw TYPE_ORDER union member"); + require("IEEE_754_TOTAL_ORDER".equals(footer.rawColumnOrders.values.get(1).member), + "raw IEEE union member"); + Statistics float32 = metadata.getRow_groups().get(0).getColumns().get(0) + .getMeta_data().getStatistics(); + require(float32.isSetMax_value() && float32.isSetMin_value(), "modern float32 bounds"); + require(Arrays.equals(float32.getMin_value(), hex("00000080")), "float32 negative-zero bytes"); + require(Arrays.equals(float32.getMax_value(), hex("00000000")), "float32 positive-zero bytes"); + require(float32.isSetNan_count() && float32.getNan_count() == 0, "present zero nan_count"); + require(float32.isSetIs_max_value_exact() && float32.isIs_max_value_exact(), + "present true exactness"); + require(float32.isSetIs_min_value_exact() && !float32.isIs_min_value_exact(), + "present false exactness"); + Statistics float16 = metadata.getRow_groups().get(0).getColumns().get(2) + .getMeta_data().getStatistics(); + require(!float16.isSetIs_max_value_exact(), "absent exactness"); + require(Arrays.equals(float16.getMin_value(), hex("0080")), "float16 negative-zero bytes"); + } + + private static void checkEvidence(Path fixture) throws Exception { + String evidence = RawFooterScanner.scan(fixture, "self-test.parquet"); + require(evidence.indexOf('\n') < 0, "one JSON object per evidence line"); + require(evidence.contains("\"evidence_version\":\"parquet-2.13-raw-footer-v3\""), + "raw evidence schema version"); + require(evidence.contains("\"schema_leaf_count\":8,\"schema_leaves\":["), + "raw schema leaf table"); + require(evidence.contains("\"path\":[\"uint16\"],\"physical_type\":\"INT32\"," + + "\"type_length\":{\"present\":true,\"value\":16}," + + "\"converted_type\":{\"present\":true,\"value\":\"UINT_16\"}"), + "raw integer legacy descriptor"); + require(evidence.contains("\"field_id\":{\"present\":true,\"value\":17}," + + "\"logical_type\":{\"present\":true,\"member\":\"INTEGER\"," + + "\"parameters\":{\"bit_width\":16,\"is_signed\":false}}"), + "raw integer logical descriptor"); + require(evidence.contains("\"path\":[\"decimal4\"],\"physical_type\":" + + "\"FIXED_LEN_BYTE_ARRAY\",\"type_length\":{\"present\":true,\"value\":4}," + + "\"converted_type\":{\"present\":true,\"value\":\"DECIMAL\"}," + + "\"scale\":{\"present\":true,\"value\":2}," + + "\"precision\":{\"present\":true,\"value\":9}"), + "raw decimal legacy descriptor"); + require(evidence.contains("\"member\":\"DECIMAL\"," + + "\"parameters\":{\"scale\":2,\"precision\":9}"), + "raw decimal logical descriptor"); + require(evidence.contains("\"path\":[\"time64\"]") + && evidence.contains("\"member\":\"TIME\"," + + "\"parameters\":{\"unit\":\"MICROS\",\"is_adjusted_to_utc\":true}") + && evidence.contains("\"member\":\"TIMESTAMP\"," + + "\"parameters\":{\"unit\":\"NANOS\",\"is_adjusted_to_utc\":false}"), + "raw time logical descriptors"); + require(evidence.contains("\"state\":\"known\",\"field_id\":2,\"wire_type\":12," + + "\"header_hex\":\"2c\"," + + "\"member\":\"IEEE_754_TOTAL_ORDER\""), + "raw IEEE union field"); + require(evidence.contains("\"name\":\"nan_count\",\"present\":true,\"value\":0"), + "raw present-zero nan count"); + require(evidence.contains("\"hex\":\"00000080\",\"byte_length\":4," + + "\"float_bits\":{\"width\":32,\"valid_width\":true,\"hex\":\"0x80000000\"}"), + "raw float32 signed-zero bit pattern"); + require(evidence.contains("\"hex\":\"010000000000f8ff\",\"byte_length\":8," + + "\"float_bits\":{\"width\":64,\"valid_width\":true," + + "\"hex\":\"0xfff8000000000001\"}"), "raw float64 NaN payload bits"); + require(evidence.contains("\"hex\":\"0080\",\"byte_length\":2," + + "\"float_bits\":{\"width\":16,\"valid_width\":true,\"hex\":\"0x8000\"}"), + "raw float16 signed-zero bit pattern"); + require(evidence.contains("\"name\":\"is_max_value_exact\",\"present\":false,\"value\":null"), + "raw absent exactness"); + require(evidence.contains("raw-java self-test \\u03c0"), "stable non-ASCII JSON escaping"); + } + + private static void checkRawColumnOrderMutations(Path fixture) throws Exception { + Path directory = Files.createTempDirectory("raw-java-orders-"); + try { + Path id1 = mutateOrderHeader(fixture, directory.resolve("id1.parquet"), 1, (byte) 0x1c); + require(RawFooterScanner.scan(id1, "id1.parquet").contains( + "\"ordinal\":1,\"schema_leaf_ordinal\":1,\"state\":\"known\"," + + "\"field_id\":1,\"wire_type\":12," + + "\"header_hex\":\"1c\"," + + "\"member\":\"TYPE_ORDER\""), "mutated raw TYPE_ORDER field ID"); + + Path id2 = mutateOrderHeader(fixture, directory.resolve("id2.parquet"), 0, (byte) 0x2c); + require(RawFooterScanner.scan(id2, "id2.parquet").contains( + "\"ordinal\":0,\"schema_leaf_ordinal\":0,\"state\":\"known\"," + + "\"field_id\":2,\"wire_type\":12," + + "\"header_hex\":\"2c\"," + + "\"member\":\"IEEE_754_TOTAL_ORDER\""), "mutated raw IEEE field ID"); + + Path unknown = mutateOrderHeader( + fixture, directory.resolve("unknown.parquet"), 0, (byte) 0x3c); + require(RawFooterScanner.scan(unknown, "unknown.parquet").contains( + "\"ordinal\":0,\"schema_leaf_ordinal\":0,\"state\":\"unknown\"," + + "\"field_id\":3," + + "\"wire_type\":12,\"header_hex\":\"3c\",\"member\":null"), + "unknown raw union field ID"); + + Path minimumId = mutateOrderExplicitId( + fixture, directory.resolve("minimum-id.parquet"), 0, Short.MIN_VALUE); + require(RawFooterScanner.scan(minimumId, "minimum-id.parquet").contains( + "\"ordinal\":0,\"schema_leaf_ordinal\":0,\"state\":\"unknown\"," + + "\"field_id\":-32768," + + "\"wire_type\":12,\"header_hex\":\"0cffff03\",\"member\":null"), + "minimum raw i16 field ID"); + + Path maximumId = mutateOrderExplicitId( + fixture, directory.resolve("maximum-id.parquet"), 0, Short.MAX_VALUE); + require(RawFooterScanner.scan(maximumId, "maximum-id.parquet").contains( + "\"ordinal\":0,\"schema_leaf_ordinal\":0,\"state\":\"unknown\"," + + "\"field_id\":32767," + + "\"wire_type\":12,\"header_hex\":\"0cfeff03\",\"member\":null"), + "maximum raw i16 field ID"); + + Path wrongType = mutateOrderHeader( + fixture, directory.resolve("wrong-type.parquet"), 0, (byte) 0x15); + require(RawFooterScanner.scan(wrongType, "wrong-type.parquet").contains( + "\"ordinal\":0,\"schema_leaf_ordinal\":0,\"state\":\"wrong_type\"," + + "\"field_id\":1," + + "\"wire_type\":8,\"header_hex\":\"15\",\"member\":null"), + "wrong raw union wire type"); + + Path empty = mutateOrderEmpty(fixture, directory.resolve("empty.parquet"), 0); + require(RawFooterScanner.scan(empty, "empty.parquet").contains( + "\"ordinal\":0,\"schema_leaf_ordinal\":0,\"state\":\"empty\"," + + "\"field_id\":null," + + "\"wire_type\":null,\"header_hex\":\"00\",\"member\":null"), + "empty raw union member"); + } finally { + deleteTree(directory); + } + } + + private static void checkSchemaFailures() throws Exception { + FileMetaData absent = SelfTestFixture.metadata().deepCopy(); + absent.unsetSchema(); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(absent), "absent schema"); + + FileMetaData empty = SelfTestFixture.metadata().deepCopy(); + empty.setSchema(List.of()); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(empty), "empty schema"); + + FileMetaData truncated = SelfTestFixture.metadata().deepCopy(); + truncated.getSchema().get(0).setNum_children(5); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(truncated), + "truncated schema topology"); + + FileMetaData orphan = SelfTestFixture.metadata().deepCopy(); + orphan.getSchema().get(0).setNum_children(3); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(orphan), + "orphan schema element"); + + FileMetaData duplicate = SelfTestFixture.metadata().deepCopy(); + duplicate.getSchema().get(4).setName("float32"); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(duplicate), + "duplicate schema path"); + + FileMetaData invalidRoot = SelfTestFixture.metadata().deepCopy(); + invalidRoot.getSchema().get(0).setNum_children(-1); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(invalidRoot), + "negative schema child count"); + + FileMetaData invalidLeaf = SelfTestFixture.metadata().deepCopy(); + invalidLeaf.getSchema().get(1).setNum_children(1); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(invalidLeaf), + "leaf with child count"); + + FileMetaData absentRows = SelfTestFixture.metadata().deepCopy(); + absentRows.unsetRow_groups(); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(absentRows), + "absent row groups"); + + FileMetaData negativeRows = SelfTestFixture.metadata().deepCopy(); + negativeRows.setNum_rows(-1); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(negativeRows), + "negative file row count"); + + FileMetaData absentRowGroupRows = SelfTestFixture.metadata().deepCopy(); + absentRowGroupRows.getRow_groups().get(0).unsetNum_rows(); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(absentRowGroupRows), + "absent row-group row count"); + + FileMetaData negativeRowGroupRows = SelfTestFixture.metadata().deepCopy(); + negativeRowGroupRows.getRow_groups().get(0).setNum_rows(-1); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(negativeRowGroupRows), + "negative row-group row count"); + + FileMetaData absentByteSize = SelfTestFixture.metadata().deepCopy(); + absentByteSize.getRow_groups().get(0).unsetTotal_byte_size(); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(absentByteSize), + "absent row-group total byte size"); + + FileMetaData absentValues = SelfTestFixture.metadata().deepCopy(); + absentValues.getRow_groups().get(0).getColumns().get(0).getMeta_data().unsetNum_values(); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(absentValues), + "absent column value count"); + + FileMetaData negativeValues = SelfTestFixture.metadata().deepCopy(); + negativeValues.getRow_groups().get(0).getColumns().get(0).getMeta_data().setNum_values(-1); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(negativeValues), + "negative column value count"); + + FileMetaData wrongPhysicalType = SelfTestFixture.metadata().deepCopy(); + wrongPhysicalType.getRow_groups().get(0).getColumns().get(0).getMeta_data() + .setType(org.apache.parquet.format.Type.INT32); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(wrongPhysicalType), + "column and leaf physical type mismatch"); + + FileMetaData unknownPath = SelfTestFixture.metadata().deepCopy(); + unknownPath.getRow_groups().get(0).getColumns().get(0).getMeta_data() + .setPath_in_schema(List.of("unknown")); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(unknownPath), + "column path absent from leaf table"); + + FileMetaData missingColumn = SelfTestFixture.metadata().deepCopy(); + missingColumn.getRow_groups().get(0).getColumns().remove(0); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(missingColumn), + "row-group column and leaf count mismatch"); + + FileMetaData wrongColumnOrder = SelfTestFixture.metadata().deepCopy(); + org.apache.parquet.format.ColumnChunk first = + wrongColumnOrder.getRow_groups().get(0).getColumns().get(0); + wrongColumnOrder.getRow_groups().get(0).getColumns().set(0, + wrongColumnOrder.getRow_groups().get(0).getColumns().get(1)); + wrongColumnOrder.getRow_groups().get(0).getColumns().set(1, first); + expect(IOException.class, () -> RawFooterScanner.validateMetadata(wrongColumnOrder), + "row-group column and leaf order mismatch"); + } + + private static void checkStructuralPreflight() throws Exception { + CompactThriftPreflight.validate(new byte[] {0}); + expect(IOException.class, () -> CompactThriftPreflight.validate( + compactFieldWithLength((byte) 0x18, 64L * 1024 * 1024 + 1)), + "oversized Compact-Thrift binary declaration"); + expect(IOException.class, () -> CompactThriftPreflight.validate( + compactListWithLength(1_000_001)), + "oversized Compact-Thrift container declaration"); + expect(IOException.class, () -> CompactThriftPreflight.validate( + new byte[] {0x18, (byte) 0x80}), "unterminated Compact-Thrift length"); + byte[] aggregateElements = new byte[] { + 0x19, 0x23, 0, 0, 0x19, 0x23, 0, 0, 0 + }; + CompactThriftPreflight.validateForTest(aggregateElements, 128, 10, 4, 10, 10); + expect(IOException.class, () -> CompactThriftPreflight.validateForTest( + aggregateElements, 128, 10, 3, 10, 10), + "oversized Compact-Thrift aggregate element count"); + byte[] aggregateBytes = new byte[] { + 0x18, 2, 0, 0, 0x18, 2, 0, 0, 0 + }; + CompactThriftPreflight.validateForTest(aggregateBytes, 128, 10, 10, 2, 4); + expect(IOException.class, () -> CompactThriftPreflight.validateForTest( + aggregateBytes, 128, 10, 10, 2, 3), + "oversized Compact-Thrift aggregate binary bytes"); + expect(JsonWriter.LimitException.class, () -> new JsonWriter(5) + .hexValue(ByteBuffer.wrap(new byte[] {0, 1}), false, false), + "hex evidence checks its exact size before growth"); + byte[] nested = new byte[261]; + Arrays.fill(nested, 0, 130, (byte) 0x1c); + Arrays.fill(nested, 130, nested.length, (byte) 0); + expect(IOException.class, () -> CompactThriftPreflight.validate(nested), + "oversized Compact-Thrift depth"); + } + + private static void checkFailures(Path fixture) throws Exception { + Path directory = Files.createTempDirectory("raw-java-negative-"); + try { + byte[] valid = Files.readAllBytes(fixture); + Path leading = directory.resolve("leading.parquet"); + byte[] badLeading = valid.clone(); + badLeading[0] = 'X'; + Files.write(leading, badLeading); + expect(IOException.class, () -> RawFooterScanner.readFooter(leading), "leading magic"); + + Path trailing = directory.resolve("trailing.parquet"); + byte[] badTrailing = valid.clone(); + badTrailing[badTrailing.length - 1] = 'X'; + Files.write(trailing, badTrailing); + expect(IOException.class, () -> RawFooterScanner.readFooter(trailing), "trailing magic"); + + Path envelope = directory.resolve("envelope.parquet"); + byte[] badEnvelope = valid.clone(); + int lengthOffset = badEnvelope.length - 8; + Arrays.fill(badEnvelope, lengthOffset, lengthOffset + 4, (byte) 0xff); + Files.write(envelope, badEnvelope); + expect(IOException.class, () -> RawFooterScanner.readFooter(envelope), "footer containment"); + + Path trailingThrift = directory.resolve("trailing-thrift.parquet"); + Files.write(trailingThrift, addTrailingFooterByte(valid)); + expect(IOException.class, () -> RawFooterScanner.readFooter(trailingThrift), + "trailing Compact-Thrift byte"); + + Path shortFile = directory.resolve("short.parquet"); + Files.write(shortFile, new byte[] {'P', 'A', 'R', '1'}); + expect(IOException.class, () -> RawFooterScanner.readFooter(shortFile), "short file"); + + Path output = directory.resolve("evidence.jsonl"); + byte[] sentinel = "unchanged\n".getBytes(StandardCharsets.UTF_8); + Files.write(output, sentinel); + expect(IOException.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", envelope.toString(), "--output", output.toString() + }), "failure before output replacement"); + require(Arrays.equals(Files.readAllBytes(output), sentinel), "failed scan leaves output unchanged"); + + expect(IOException.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", fixture.toString(), "--output", output.toString() + }, 128), "incremental evidence output limit"); + require(Arrays.equals(Files.readAllBytes(output), sentinel), + "evidence limit leaves output unchanged"); + + Path missingRowCount = directory.resolve("missing-row-count.parquet"); + SelfTestFixture.writeMissingRequired(missingRowCount, true, false); + expect(Exception.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", missingRowCount.toString(), "--output", output.toString() + }), "missing row-group count before output"); + require(Arrays.equals(Files.readAllBytes(output), sentinel), + "missing row-group count leaves output unchanged"); + + Path missingValueCount = directory.resolve("missing-value-count.parquet"); + SelfTestFixture.writeMissingRequired(missingValueCount, false, true); + expect(Exception.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", missingValueCount.toString(), "--output", output.toString() + }), "missing column value count before output"); + require(Arrays.equals(Files.readAllBytes(output), sentinel), + "missing column value count leaves output unchanged"); + + FileMetaData negativeRowMetadata = SelfTestFixture.metadata().deepCopy(); + negativeRowMetadata.getRow_groups().get(0).setNum_rows(-1); + Path negativeRowCount = directory.resolve("negative-row-count.parquet"); + SelfTestFixture.write(negativeRowCount, negativeRowMetadata); + expect(IOException.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", negativeRowCount.toString(), "--output", output.toString() + }), "negative row-group count before output"); + require(Arrays.equals(Files.readAllBytes(output), sentinel), + "negative row-group count leaves output unchanged"); + + FileMetaData negativeValueMetadata = SelfTestFixture.metadata().deepCopy(); + negativeValueMetadata.getRow_groups().get(0).getColumns().get(0).getMeta_data() + .setNum_values(-1); + Path negativeValueCount = directory.resolve("negative-value-count.parquet"); + SelfTestFixture.write(negativeValueCount, negativeValueMetadata); + expect(IOException.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", negativeValueCount.toString(), "--output", output.toString() + }), "negative column value count before output"); + require(Arrays.equals(Files.readAllBytes(output), sentinel), + "negative column value count leaves output unchanged"); + + String fixtureHash = sha256(Files.readAllBytes(fixture)); + expect(IOException.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", fixture.toString(), "--output", fixture.toString() + }), "direct output alias"); + require(fixtureHash.equals(sha256(Files.readAllBytes(fixture))), + "direct output alias leaves input unchanged"); + + Path hardlink = directory.resolve("hardlink.parquet"); + Files.createLink(hardlink, fixture); + expect(IOException.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", fixture.toString(), "--output", hardlink.toString() + }), "hard-link output alias"); + require(fixtureHash.equals(sha256(Files.readAllBytes(fixture))) + && fixtureHash.equals(sha256(Files.readAllBytes(hardlink))), + "hard-link output alias leaves input unchanged"); + + Path symlink = directory.resolve("symlink.parquet"); + Files.createSymbolicLink(symlink, fixture); + expect(IOException.class, () -> RawFooterScanner.execute(new String[] { + "scan", "--input", fixture.toString(), "--output", symlink.toString() + }), "symbolic-link output alias"); + require(fixtureHash.equals(sha256(Files.readAllBytes(fixture))) + && fixtureHash.equals(sha256(Files.readAllBytes(symlink))), + "symbolic-link output alias leaves input unchanged"); + + Path changing = directory.resolve("changing.parquet"); + Files.copy(fixture, changing); + expect(IOException.class, () -> RawFooterScanner.readFooter(changing, () -> { + try (RandomAccessFile changed = new RandomAccessFile(changing.toFile(), "rw")) { + changed.seek(0); + changed.write('X'); + } catch (IOException exception) { + throw new IllegalStateException("cannot mutate snapshot fixture", exception); + } + }), "input snapshot race"); + + Path oversized = directory.resolve("oversized-footer.parquet"); + writeOversizedFooterEnvelope(oversized); + expect(IOException.class, () -> RawFooterScanner.readFooter(oversized), + "footer resource limit"); + + Path oversizedDeclaration = directory.resolve("oversized-declaration.parquet"); + SelfTestFixture.writeRawFooter(oversizedDeclaration, + compactFieldWithLength((byte) 0x18, 64L * 1024 * 1024 + 1)); + expect(IOException.class, () -> RawFooterScanner.readFooter(oversizedDeclaration), + "pre-decode binary declaration limit"); + } finally { + deleteTree(directory); + } + } + + private static Path mutateOrderHeader(Path fixture, Path output, int ordinal, byte header) + throws Exception { + RawFooterScanner.Footer footer = RawFooterScanner.readFooter(fixture); + RawFooterScanner.RawColumnOrder order = footer.rawColumnOrders.values.get(ordinal); + byte[] bytes = Files.readAllBytes(fixture); + int footerStart = bytes.length - 8 - footer.footerLength; + int offset = footerStart + order.headerOffset; + require((bytes[offset] & 0x0f) == 0x0c, "source order uses a raw struct header"); + bytes[offset] = header; + Files.write(output, bytes); + return output; + } + + private static byte[] compactFieldWithLength(byte fieldHeader, long length) { + byte[] encoded = unsignedVarint(length); + byte[] bytes = new byte[encoded.length + 2]; + bytes[0] = fieldHeader; + System.arraycopy(encoded, 0, bytes, 1, encoded.length); + bytes[bytes.length - 1] = 0; + return bytes; + } + + private static byte[] compactListWithLength(long length) { + byte[] encoded = unsignedVarint(length); + byte[] bytes = new byte[encoded.length + 3]; + bytes[0] = 0x19; + bytes[1] = (byte) 0xf3; + System.arraycopy(encoded, 0, bytes, 2, encoded.length); + bytes[bytes.length - 1] = 0; + return bytes; + } + + private static byte[] unsignedVarint(long value) { + byte[] bytes = new byte[10]; + int length = 0; + do { + int next = (int) (value & 0x7f); + value >>>= 7; + bytes[length++] = (byte) (value == 0 ? next : next | 0x80); + } while (value != 0); + return Arrays.copyOf(bytes, length); + } + + private static Path mutateOrderEmpty(Path fixture, Path output, int ordinal) throws Exception { + RawFooterScanner.Footer footer = RawFooterScanner.readFooter(fixture); + RawFooterScanner.RawColumnOrder order = footer.rawColumnOrders.values.get(ordinal); + byte[] bytes = Files.readAllBytes(fixture); + int footerStart = bytes.length - 8 - footer.footerLength; + int offset = footerStart + order.headerOffset; + require((bytes[offset] & 0x0f) == 0x0c && bytes[offset + 1] == 0, + "source order uses an empty struct payload"); + byte[] changed = new byte[bytes.length - 2]; + System.arraycopy(bytes, 0, changed, 0, offset); + System.arraycopy(bytes, offset + 2, changed, offset, bytes.length - offset - 2); + writeLittleEndianInt(changed, changed.length - 8, footer.footerLength - 2); + Files.write(output, changed); + return output; + } + + private static Path mutateOrderExplicitId( + Path fixture, + Path output, + int ordinal, + short fieldId + ) throws Exception { + RawFooterScanner.Footer footer = RawFooterScanner.readFooter(fixture); + RawFooterScanner.RawColumnOrder order = footer.rawColumnOrders.values.get(ordinal); + byte[] bytes = Files.readAllBytes(fixture); + int footerStart = bytes.length - 8 - footer.footerLength; + int offset = footerStart + order.headerOffset; + require((bytes[offset] & 0x0f) == 0x0c, "source order uses a raw struct header"); + int encodedId = ((fieldId << 1) ^ (fieldId >> 15)) & 0xffff; + byte[] header = new byte[4]; + int headerLength = 1; + header[0] = 0x0c; + do { + int next = encodedId & 0x7f; + encodedId >>>= 7; + header[headerLength++] = (byte) (encodedId == 0 ? next : next | 0x80); + } while (encodedId != 0); + byte[] changed = new byte[bytes.length + headerLength - 1]; + System.arraycopy(bytes, 0, changed, 0, offset); + System.arraycopy(header, 0, changed, offset, headerLength); + System.arraycopy(bytes, offset + 1, changed, offset + headerLength, + bytes.length - offset - 1); + writeLittleEndianInt(changed, changed.length - 8, + footer.footerLength + headerLength - 1); + Files.write(output, changed); + return output; + } + + private static void writeOversizedFooterEnvelope(Path output) throws IOException { + int footerLength = 64 * 1024 * 1024 + 1; + try (RandomAccessFile file = new RandomAccessFile(output.toFile(), "rw")) { + file.setLength((long) footerLength + 12); + file.seek(0); + file.write(new byte[] {'P', 'A', 'R', '1'}); + file.seek((long) footerLength + 4); + file.write(footerLength & 0xff); + file.write((footerLength >>> 8) & 0xff); + file.write((footerLength >>> 16) & 0xff); + file.write((footerLength >>> 24) & 0xff); + file.write(new byte[] {'P', 'A', 'R', '1'}); + } + } + + private static String sha256(byte[] bytes) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes); + StringBuilder hex = new StringBuilder(digest.length * 2); + for (byte value : digest) { + hex.append(String.format("%02x", value & 0xff)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + private static void deleteTree(Path directory) throws IOException { + try (java.util.stream.Stream paths = Files.walk(directory)) { + paths.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.delete(path); + } catch (IOException exception) { + throw new IllegalStateException("cannot clean self-test path " + path, exception); + } + }); + } + } + + private static byte[] addTrailingFooterByte(byte[] valid) { + int lengthOffset = valid.length - 8; + int footerLength = littleEndianInt(valid, lengthOffset); + byte[] changed = new byte[valid.length + 1]; + System.arraycopy(valid, 0, changed, 0, lengthOffset); + changed[lengthOffset] = 0; + writeLittleEndianInt(changed, lengthOffset + 1, footerLength + 1); + System.arraycopy(valid, valid.length - 4, changed, changed.length - 4, 4); + return changed; + } + + private static int littleEndianInt(byte[] bytes, int offset) { + return (bytes[offset] & 0xff) + | ((bytes[offset + 1] & 0xff) << 8) + | ((bytes[offset + 2] & 0xff) << 16) + | ((bytes[offset + 3] & 0xff) << 24); + } + + private static void writeLittleEndianInt(byte[] bytes, int offset, int value) { + bytes[offset] = (byte) value; + bytes[offset + 1] = (byte) (value >>> 8); + bytes[offset + 2] = (byte) (value >>> 16); + bytes[offset + 3] = (byte) (value >>> 24); + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int index = 0; index < bytes.length; index++) { + int high = Character.digit(value.charAt(index * 2), 16); + int low = Character.digit(value.charAt(index * 2 + 1), 16); + bytes[index] = (byte) ((high << 4) | low); + } + return bytes; + } + + private static void expect( + Class expected, + ThrowingAction action, + String description + ) throws Exception { + try { + action.run(); + } catch (Throwable exception) { + if (expected.isInstance(exception)) { + return; + } + throw new AssertionError(description + " raised " + exception.getClass().getName(), exception); + } + throw new AssertionError(description + " did not fail"); + } + + private static void require(boolean condition, String description) { + if (!condition) { + throw new AssertionError(description); + } + } +} diff --git a/test/conformance/n6/oracles/raw-java/toolchain.env b/test/conformance/n6/oracles/raw-java/toolchain.env new file mode 100644 index 0000000..bda23ce --- /dev/null +++ b/test/conformance/n6/oracles/raw-java/toolchain.env @@ -0,0 +1,29 @@ +RAW_JAVA_PLAN_SHA256=15adf34af765a3300d8a49ced73532ed02b7e4edee5764453c58e53fcf12c304 +RAW_JAVA_FORMAT_COMMIT=c47e2a66e88943fc46fde1b028a9432f14fdf5c0 +RAW_JAVA_IDL_SHA256=53bb8fc9b96469d7ca694121ead839e449e5156d7bf79f0df728cdd72796df38 +RAW_JAVA_THRIFT_VERSION=0.23.0 +RAW_JAVA_THRIFT_SOURCE_URL=https://archive.apache.org/dist/thrift/0.23.0/thrift-0.23.0.tar.gz +RAW_JAVA_THRIFT_SOURCE_SHA256=1859d932d2ae1f13d16c5a196931208c116310a5ff50f2bfd11d3db03be8f46f +RAW_JAVA_HOMEBREW_FORMULA_COMMIT=bd296b14f19462baf03d5d96920209087ca99fa0 +RAW_JAVA_HOMEBREW_FORMULA_URL=https://raw.githubusercontent.com/Homebrew/homebrew-core/bd296b14f19462baf03d5d96920209087ca99fa0/Formula/t/thrift.rb +RAW_JAVA_HOMEBREW_FORMULA_SHA256=3929691e8a327dd0e61f41c8d7a6cccdd30ab1213b01a73d0148b195d752b209 +RAW_JAVA_BOTTLE_DARWIN_ARM64_SEQUOIA_SHA256=dd6ed015e1b7a980c3dfa2b0dd1c01d563a8cf73bdb7f3de87d0cc1656fc1e1b +RAW_JAVA_COMPILER_DARWIN_ARM64_SEQUOIA_SHA256=5ee94e75371f7d0b2467db3acdb67b8b3814fcae3748c0ef078a490a15c57e11 +RAW_JAVA_GENERATED_MANIFEST_SHA256=f738c7346ad1dd70faafd54815f829b8587a2b0397ff6b6e6710a3a7276cac09 +RAW_JAVA_LIBTHRIFT_URL=https://repo.maven.apache.org/maven2/org/apache/thrift/libthrift/0.23.0/libthrift-0.23.0.jar +RAW_JAVA_LIBTHRIFT_SHA256=8b41b67a5ff13c371ab18b6d34506121dcecf11372829f7d50115cfb1bf72d42 +RAW_JAVA_LIBTHRIFT_POM_URL=https://repo.maven.apache.org/maven2/org/apache/thrift/libthrift/0.23.0/libthrift-0.23.0.pom +RAW_JAVA_LIBTHRIFT_POM_SHA256=eeadf7b9d1e22ac01985fe552384ceecf44018cae93e7a23d6b0466f856330f3 +RAW_JAVA_SLF4J_API_URL=https://repo.maven.apache.org/maven2/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar +RAW_JAVA_SLF4J_API_SHA256=d3ef575e3e4979678dc01bf1dcce51021493b4d11fb7f1be8ad982877c16a1c0 +RAW_JAVA_SLF4J_NOP_URL=https://repo.maven.apache.org/maven2/org/slf4j/slf4j-nop/1.7.36/slf4j-nop-1.7.36.jar +RAW_JAVA_SLF4J_NOP_SHA256=c214958b07816cb4412b30c7bdbd4308ffdc6ba2a83767b8f3a9229cbd9274d6 +RAW_JAVA_JDK_VENDOR=Eclipse-Adoptium-Temurin +RAW_JAVA_JDK_VERSION=21.0.8+9 +RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_URL=https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.8%2B9/OpenJDK21U-jdk_aarch64_mac_hotspot_21.0.8_9.tar.gz +RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_SHA256=59422c2292ae4e76b87e00d8808dbe49cffa39af731e08bb0292ddb0af4e0261 +RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_JAVA_SHA256=0045ae168ee132bbf469a26fb17dac6d1dee431c9b7826474f3b6ee574a997c9 +RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_JAVAC_SHA256=7be7937fc6bae0ca89f0866f9ce94fc40a935dfb87806d3c701eca3402cfb90a +RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_RELEASE_SHA256=8e98b265f9a6fd3db04d2535108497897e87f1b3821270cf10ffa937463dc2ee +RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_TREE_SHA256=d595de66a27187223eb987765fc6c9c341d509ecd15de704656a2980bc6217bc +RAW_JAVA_JAVA_RELEASE=11 diff --git a/test/conformance/n6/runtests.jl b/test/conformance/n6/runtests.jl new file mode 100644 index 0000000..6ea1c52 --- /dev/null +++ b/test/conformance/n6/runtests.jl @@ -0,0 +1,2699 @@ +using SHA +using Test +using TOML + +const N6_DIR = @__DIR__ +const REPO_DIR = normpath(joinpath(N6_DIR, "..", "..", "..")) +const SHA256_PATTERN = r"^[0-9a-f]{64}$" +const GIT_PATTERN = r"^[0-9a-f]{40}$" +const CASE_PATTERN = r"^[a-z0-9]+(?:[._-][a-z0-9]+)*$" +const CPYTHON_312_DISTRIBUTION_URL = "https://github.com/astral-sh/" * + "python-build-standalone/releases/download/20250115/" * + "cpython-3.12.8%2B20250115-aarch64-apple-darwin-" * + "install_only_stripped.tar.gz" +const CPYTHON_CLEAN_TREE_POLICY = "extract-strip-site-packages-bytecode-v1" +const CPYTHON_314_DISTRIBUTION_URL = "https://github.com/astral-sh/" * + "python-build-standalone/releases/download/20260127/" * + "cpython-3.14.2%2B20260127-aarch64-apple-darwin-" * + "install_only_stripped.tar.gz" +const VALIDATOR_CLEAN_TREE_POLICY = + "extract-strip-site-packages-bytecode-v1" +const CPYTHON_DISTRIBUTION_ARCHIVE_LIMIT = 128 * 1024 * 1024 +const VALIDATOR_DISTRIBUTION_ARCHIVE_LIMIT = 64 * 1024 * 1024 +const CONTROL_FILE_LIMIT = 64 * 1024 * 1024 +const RAW_JAVA_DOWNLOAD_LIMIT = 256 * 1024 * 1024 +const RAW_JAVA_DOWNLOAD_TOTAL_LIMIT = 256 * 1024 * 1024 +const VALIDATOR_TREE_ENTRY_LIMIT = 5_000 +const VALIDATOR_TREE_FILE_LIMIT = 64 * 1024 * 1024 +const VALIDATOR_TREE_TOTAL_LIMIT = 128 * 1024 * 1024 +const JDK_TREE_ENTRY_LIMIT = 2_000 +const JDK_TREE_FILE_LIMIT = 256 * 1024 * 1024 +const JDK_TREE_TOTAL_LIMIT = 512 * 1024 * 1024 +const GATE_EXECUTABLE_FILES = Set([ + "test/conformance/n6/oracles/arrow-rs/build/" * + "parquet-jl-n6-arrow-rs-metadata", +]) + +struct FileSnapshot + path::String + payload::Vector{UInt8} + sha256::String +end + +function file_sha256(file::AbstractString) + return bytes2hex(open(SHA.sha256, file)) +end + +function require_gate(condition::Bool, message::AbstractString) + condition || error(message) + return +end + +function read_bounded_regular_bytes(file::AbstractString, limit::Int) + limit > 0 || throw(ArgumentError("file byte limit must be positive")) + io = open(file, "r") + try + before = stat(io) + 0 < before.size <= limit || error("input has an invalid byte size: $file") + payload = read(io, limit + 1) + length(payload) <= limit || error("input exceeds its byte limit: $file") + length(payload) == before.size || error("input changed size while read: $file") + after = stat(io) + identity(value) = (value.device, value.inode, value.size, value.mtime, + value.ctime) + identity(before) == identity(after) || + error("input changed while read: $file") + return payload + finally + close(io) + end +end + +function file_snapshot(relative::AbstractString) + safe_relative(relative) || error("unsafe control path: $relative") + path = checked_file(REPO_DIR, relative) + payload = read_bounded_regular_bytes(path, CONTROL_FILE_LIMIT) + return FileSnapshot(path, payload, bytes2hex(SHA.sha256(payload))) +end + +function parse_toml_snapshot(snapshot::FileSnapshot) + return TOML.parse(String(copy(snapshot.payload))) +end + +function control_snapshot(snapshots::Dict{String,FileSnapshot}, + relative::AbstractString) + haskey(snapshots, relative) || error("control snapshot is absent: $relative") + return snapshots[relative] +end + +function load_control_snapshots(manifest) + paths = Set{String}([ + manifest["plan_file"], + manifest["capabilities_file"], + manifest["fixture_manifest_file"], + manifest["corpus_manifest_file"], + manifest["evidence_schema_file"], + manifest["artifact_manifest_file"], + manifest["model_producer_descriptor_file"], + manifest["parquet_jl_producer_descriptor_file"], + "thrift/parquet.thrift", + "test/conformance/n6/model/cases.toml", + "test/conformance/n6/toolchains/pyarrow.toml", + "test/conformance/n6/toolchains/duckdb.toml", + "test/conformance/n6/oracles/arrow-rs/toolchain.toml", + "test/conformance/n6/oracles/parquet-java/toolchain.toml", + ]) + snapshots = Dict{String,FileSnapshot}() + for relative in paths + relative isa String || error("control path is not a string") + snapshots[relative] = file_snapshot(relative) + end + require_gate(control_snapshot(snapshots, + "thrift/parquet.thrift").sha256 == toolchain_artifact(manifest, + "raw-java", "parquet-2.13-idl"), + "local raw scanner IDL hash differs") + artifact_manifest = control_snapshot(snapshots, + manifest["artifact_manifest_file"]) + for (relative, digest) in artifact_entries(artifact_manifest.payload) + repository_relative = "test/conformance/n6/" * relative + snapshot = get!(() -> file_snapshot(repository_relative), snapshots, + repository_relative) + require_gate(snapshot.sha256 == digest, + "artifact hash differs: $repository_relative") + end + producer_descriptor = parse_toml_snapshot(control_snapshot(snapshots, + manifest["parquet_jl_producer_descriptor_file"])) + for item in producer_descriptor["file"] + relative = item["path"] + snapshot = get!(() -> file_snapshot(relative), snapshots, relative) + require_gate(snapshot.sha256 == item["sha256"], + "Parquet.jl producer file hash differs: $relative") + end + producer_manifest = producer_descriptor["manifest_file"] + snapshots["Manifest.toml"] = control_snapshot(snapshots, producer_manifest) + return snapshots +end + +function load_generated_snapshots!(snapshots, fixtures) + for generated in fixtures["generated_case"] + relative = "test/conformance/n6/" * generated["output_file"] + snapshot = file_snapshot(relative) + require_gate(snapshot.sha256 == generated["output_sha256"], + "generated fixture hash differs: " * generated["id"]) + snapshots[relative] = snapshot + end + return +end + +function load_evidence_snapshots!(snapshots, manifest) + for evidence in vcat(manifest["frozen_evidence"], + manifest["planned_evidence"]) + relative = evidence["file"] + if !ispath(joinpath(REPO_DIR, relative)) + evidence["storage"] == "gate-generated" || + error("evidence file is absent: $relative") + continue + end + snapshot = file_snapshot(relative) + if evidence["status"] == "verified" && + evidence["storage"] == "checked-in" + require_gate(snapshot.sha256 == evidence["sha256"], + "frozen evidence hash differs: " * evidence["id"]) + end + snapshots[relative] = snapshot + end + return +end + +function load_oracle_build_snapshots!(snapshots, manifest) + validate_oracle_build_root(joinpath(N6_DIR, "oracles", "arrow-rs"), + "Arrow Rust"; reject_nested_symlinks=true) + validate_oracle_build_root(joinpath(N6_DIR, "oracles", "parquet-java"), + "Parquet Java"; reject_nested_symlinks=true) + files = [ + ("test/conformance/n6/oracles/arrow-rs/build/" * + "parquet-jl-n6-arrow-rs-metadata", "rust-arrow-rs", + "parquet-jl-n6-arrow-rs-metadata"), + ("test/conformance/n6/oracles/parquet-java/build/artifacts/" * + "hadoop-client-api-3.3.0.jar", "parquet-java-interop", + "hadoop-client-api-3.3.0.jar"), + ("test/conformance/n6/oracles/parquet-java/build/artifacts/" * + "hadoop-client-runtime-3.3.0.jar", "parquet-java-interop", + "hadoop-client-runtime-3.3.0.jar"), + ("test/conformance/n6/oracles/parquet-java/build/artifacts/" * + "parquet-cli-1.17.1-runtime.jar", "parquet-java-interop", + "parquet-cli-1.17.1-runtime.jar"), + ("test/conformance/n6/oracles/parquet-java/build/artifacts/" * + "parquet-java-n6-harness.jar", "parquet-java-interop", + "parquet-java-n6-harness.jar"), + ] + for (relative, toolchain, artifact) in files + snapshot = file_snapshot(relative) + require_gate(snapshot.sha256 == toolchain_artifact(manifest, + toolchain, artifact), "oracle build artifact differs: $artifact") + snapshots[relative] = snapshot + end + return +end + +function set_repository_snapshot_modes!(root::AbstractString, locked::Bool) + directories = String[] + for (directory, children, files) in walkdir(root; follow_symlinks=false) + push!(directories, directory) + for name in files + file = joinpath(directory, name) + islink(file) && error("gate snapshot file is a symbolic link") + chmod(file, locked ? 0o400 : 0o600) + end + for name in children + islink(joinpath(directory, name)) && + error("gate snapshot directory is a symbolic link") + end + end + for directory in reverse(directories) + chmod(directory, locked ? 0o500 : 0o700) + end + return +end + +function with_gate_repository(f, snapshots; + writable_directories::Vector{String}=String[]) + return mktempdir() do directory + root = joinpath(directory, "repository") + mkdir(root) + root = realpath(root) + for (relative, snapshot) in snapshots + safe_relative(relative) || error("unsafe gate snapshot path: $relative") + target = joinpath(root, relative) + mkpath(dirname(target)) + open(target, "w") do io + write(io, snapshot.payload) + return + end + end + set_repository_snapshot_modes!(root, true) + for relative in GATE_EXECUTABLE_FILES + haskey(snapshots, relative) || continue + chmod(gate_file(root, relative), 0o500) + end + writable_inventory = Dict{String,Vector{String}}() + for relative in writable_directories + safe_relative(relative) || + error("unsafe writable snapshot path: $relative") + haskey(writable_inventory, relative) && + error("duplicate writable snapshot path: $relative") + target = checked_directory(root, relative) + writable_inventory[relative] = sort(readdir(target)) + chmod(target, 0o700) + end + try + result = f(root) + for (relative, inventory) in writable_inventory + target = checked_directory(root, relative) + require_gate(sort(readdir(target)) == inventory, + "writable gate snapshot inventory changed: $relative") + end + for (relative, snapshot) in snapshots + require_gate(file_sha256(gate_file(root, relative)) == + snapshot.sha256, "gate snapshot changed: $relative") + require_gate(file_sha256(snapshot.path) == snapshot.sha256, + "canonical input changed: $relative") + end + return result + finally + set_repository_snapshot_modes!(root, false) + end + end +end + +function gate_file(root::AbstractString, relative::AbstractString) + return checked_file(root, relative) +end + +function require_keys(value::AbstractDict, allowed, required=allowed) + actual = Set(keys(value)) + @test isempty(setdiff(actual, Set(allowed))) + @test isempty(setdiff(Set(required), actual)) + return +end + +function safe_relative(file::AbstractString) + isempty(file) && return false + isabspath(file) && return false + occursin('\\', file) && return false + all(character -> isascii(character) && + (isletter(character) || isdigit(character) || + character in ('_', '.', '+', '@', '=', '-', '/')), file) || return false + parts = split(file, '/') + any(part -> isempty(part) || part in (".", ".."), parts) && return false + any(character -> character < ' ', file) && return false + return normpath(file) == file +end + +function safe_evidence_file(file::AbstractString) + prefix = "test/conformance/n6/evidence/" + return safe_relative(file) && startswith(file, prefix) && + length(file) > length(prefix) && endswith(basename(file), ".jsonl") && + basename(file) != ".jsonl" +end + +function safe_generated_output(file::AbstractString) + prefix = "generated/" + return safe_relative(file) && startswith(file, prefix) && + length(file) > length(prefix) && endswith(basename(file), ".parquet") && + basename(file) != ".parquet" +end + +function checked_file(root::AbstractString, relative::AbstractString) + safe_relative(relative) || error("unsafe relative path: $relative") + root_path = realpath(root) + candidate = joinpath(root_path, relative) + isfile(candidate) || error("missing file: $relative") + islink(candidate) && error("pinned file is a symbolic link: $relative") + resolved = realpath(candidate) + startswith(resolved, root_path * Base.Filesystem.path_separator) || + error("path escapes its source root: $relative") + return resolved +end + +function checked_directory_files(root::AbstractString, expected::Vector{String}) + require_gate(sort(readdir(root)) == sort(expected), + "directory contents differ from the pinned inventory: $root") + return [checked_file(root, name) for name in expected] +end + +function artifact_entries(payload::Vector{UInt8}) + entries = Pair{String,String}[] + for (line_number, line) in enumerate(eachline(IOBuffer(payload))) + match_result = match(r"^([0-9a-f]{64}) ([A-Za-z0-9._+@=/\-]+)$", line) + match_result === nothing && error("invalid artifact line $line_number") + digest, relative = match_result.captures + safe_relative(relative) || error("unsafe artifact path: $relative") + push!(entries, relative => digest) + end + paths = first.(entries) + paths == sort(paths) || error("artifact paths are not sorted") + length(paths) == length(unique(paths)) || error("duplicate artifact path") + return entries +end + +function validate_oracle_build_root(oracle_dir::AbstractString, + oracle_name::AbstractString; reject_nested_symlinks::Bool=false) + oracle_root = realpath(oracle_dir) + build = joinpath(oracle_root, "build") + (ispath(build) || islink(build)) || return build + islink(build) && error("$oracle_name build root is a symbolic link") + isdir(build) || error("$oracle_name build root is not a directory") + resolved = realpath(build) + startswith(resolved, oracle_root * Base.Filesystem.path_separator) || + error("$oracle_name build root escapes its oracle directory") + if reject_nested_symlinks + for (directory, directories, files) in walkdir(resolved; + follow_symlinks=false) + for name in vcat(directories, files) + nested = joinpath(directory, name) + islink(nested) || continue + error("$oracle_name build cache contains a symbolic link: " * + relpath(nested, resolved)) + end + end + end + return resolved +end + +function validate_raw_build_root(raw_dir::AbstractString) + return validate_oracle_build_root(raw_dir, "raw Java") +end + +function intended_artifacts(root::AbstractString=N6_DIR; + mutable_files::Set{String}=Set{String}()) + paths = String[] + ignored_builds = Set([ + validate_oracle_build_root(joinpath(root, "oracles", "arrow-rs"), + "Arrow Rust"; reject_nested_symlinks=true), + validate_oracle_build_root(joinpath(root, "oracles", "raw-java"), + "raw Java"), + validate_oracle_build_root(joinpath(root, "oracles", "parquet-java"), + "Parquet Java"; reject_nested_symlinks=true), + ]) + for (directory, directories, files) in walkdir(root; follow_symlinks=false) + for name in directories + absolute = joinpath(directory, name) + normpath(absolute) in ignored_builds && continue + islink(absolute) && error("unlisted N6 directory symbolic link: " * + relpath(absolute, root)) + end + filter!(name -> normpath(joinpath(directory, name)) ∉ ignored_builds, + directories) + for name in files + absolute = joinpath(directory, name) + islink(absolute) && error("unlisted N6 file symbolic link: " * + relpath(absolute, root)) + relative = relpath(absolute, root) + relative in ("artifacts.sha256", "manifest.toml") && continue + portable = replace(relative, Base.Filesystem.path_separator => '/') + portable in mutable_files && continue + push!(paths, portable) + end + end + sort!(paths) + return paths +end + +function tree_sha256(root::AbstractString; excluded::Union{Nothing,String}=nothing) + root_path = realpath(root) + entries = String[] + for (directory, directories, files) in walkdir(root_path; follow_symlinks=false) + for name in vcat(directories, files) + absolute = joinpath(directory, name) + (isfile(absolute) || islink(absolute)) || continue + relative = replace(relpath(absolute, root_path), Base.Filesystem.path_separator => '/') + if excluded !== nothing && + (relative == excluded || startswith(relative, excluded * "/")) + continue + end + push!(entries, relative) + end + end + sort!(entries) + buffer = IOBuffer() + for relative in entries + absolute = joinpath(root_path, relative) + if islink(absolute) + write(buffer, "L\0", relative, "\0", readlink(absolute), '\n') + else + write(buffer, "F\0", relative, "\0", file_sha256(absolute), '\n') + end + end + return bytes2hex(SHA.sha256(take!(buffer))) +end + +function validate_bounded_tree(root::AbstractString; max_entries::Int, + max_file_bytes::Int, max_total_bytes::Int) + max_entries > 0 || throw(ArgumentError("tree entry limit must be positive")) + max_file_bytes > 0 || + throw(ArgumentError("tree file byte limit must be positive")) + max_total_bytes >= max_file_bytes || + throw(ArgumentError("tree total byte limit is too small")) + islink(root) && error("tree root is a symbolic link: $root") + isdir(root) || error("tree root is not a directory: $root") + root_path = realpath(root) + entry_count = 0 + total_bytes = 0 + for (directory, directories, files) in walkdir(root_path; + follow_symlinks=false) + for name in vcat(directories, files) + absolute = joinpath(directory, name) + relative = replace(relpath(absolute, root_path), + Base.Filesystem.path_separator => '/') + entry_count < max_entries || + error("tree exceeds its entry limit: $root_path") + entry_count += 1 + metadata = lstat(absolute) + kind = metadata.mode & Base.Filesystem.S_IFMT + if kind == Base.Filesystem.S_IFLNK + target = readlink(absolute) + isabspath(target) && + error("tree has an absolute symbolic link: $relative") + target_path = normpath(joinpath(dirname(absolute), target)) + startswith(target_path, + root_path * Base.Filesystem.path_separator) || + error("tree symbolic link escapes its root: $relative") + ispath(absolute) || + error("tree has a broken symbolic link: $relative") + resolved = realpath(absolute) + startswith(resolved, + root_path * Base.Filesystem.path_separator) || + error("tree symbolic link resolves outside its root: $relative") + elseif kind == Base.Filesystem.S_IFDIR + continue + elseif kind == Base.Filesystem.S_IFREG + metadata.size <= max_file_bytes || + error("tree file exceeds its byte limit: $relative") + metadata.size <= max_total_bytes - total_bytes || + error("tree exceeds its total byte limit: $root_path") + total_bytes += metadata.size + else + error("tree contains a special file: $relative") + end + end + end + return (entries=entry_count, bytes=total_bytes, + sha256=tree_sha256(root_path)) +end + +function toolchain_artifact(manifest, toolchain_id::AbstractString, + artifact_name::AbstractString) + toolchain = only(filter(item -> item["id"] == toolchain_id, + manifest["toolchain"])) + artifact = only(filter(item -> item["name"] == artifact_name, + toolchain["artifacts"])) + return artifact["sha256"] +end + +function validate_manifest_header(manifest) + require_keys(manifest, [ + "manifest_version", "gate", "status", "plan_file", "plan_sha256", + "capabilities_file", "capabilities_sha256", "fixture_manifest_file", + "fixture_manifest_sha256", "corpus_manifest_file", + "corpus_manifest_sha256", "evidence_schema_file", + "evidence_schema_sha256", "artifact_manifest_file", + "artifact_manifest_sha256", "model_producer_descriptor_file", + "model_producer_descriptor_sha256", + "parquet_jl_producer_descriptor_file", + "parquet_jl_producer_descriptor_sha256", + "parquet_jl_source_composite_sha256", "supported_platforms", + "publication_authorized", "oracle_lock_authorized", "evidence_limits", + "source", "toolchain", "frozen_model", "frozen_evidence", + "planned_evidence", + ]) + @test manifest["manifest_version"] == 1 + @test manifest["gate"] == "n6-a-preproduction" + @test manifest["status"] == "preproduction" + @test manifest["supported_platforms"] == ["macos-15-arm64"] + @test manifest["publication_authorized"] === false + @test manifest["oracle_lock_authorized"] === false + @test all(field -> manifest[field] isa String, ( + "gate", "status", "plan_file", "plan_sha256", "capabilities_file", + "capabilities_sha256", "fixture_manifest_file", + "fixture_manifest_sha256", "corpus_manifest_file", + "corpus_manifest_sha256", "evidence_schema_file", + "evidence_schema_sha256", "artifact_manifest_file", + "artifact_manifest_sha256", "model_producer_descriptor_file", + "model_producer_descriptor_sha256", + "parquet_jl_producer_descriptor_file", + "parquet_jl_producer_descriptor_sha256", + "parquet_jl_source_composite_sha256")) + @test safe_relative(manifest["model_producer_descriptor_file"]) + @test safe_relative(manifest["parquet_jl_producer_descriptor_file"]) + @test all(value -> value isa String, manifest["supported_platforms"]) + limits = manifest["evidence_limits"] + require_keys(limits, ["max_inputs", "max_file_bytes", "max_total_bytes", + "max_line_bytes", "max_records_per_input", "max_records_total"]) + @test all(value -> value isa Int64, values(limits)) + @test limits["max_inputs"] > 0 + @test limits["max_line_bytes"] > 0 + @test limits["max_file_bytes"] >= limits["max_line_bytes"] + @test limits["max_total_bytes"] >= limits["max_file_bytes"] + @test limits["max_records_per_input"] > 0 + @test limits["max_records_total"] >= limits["max_records_per_input"] + for key in keys(manifest) + endswith(key, "_sha256") || continue + @test occursin(SHA256_PATTERN, manifest[key]) + end + return +end + +function validate_manifest_sources(sources) + for source in sources + require_keys(source, ["id", "url", "version", "tag", "tag_revision", + "revision", "status", "root_env", "files"]) + @test all(field -> source[field] isa String, ( + "id", "url", "version", "tag", "tag_revision", "revision", + "status", "root_env")) + @test occursin(CASE_PATTERN, source["id"]) + @test source["status"] in ("verified", "planned") + @test startswith(source["url"], "https://github.com/") + @test occursin(GIT_PATTERN, source["revision"]) + @test isempty(source["tag_revision"]) || + occursin(GIT_PATTERN, source["tag_revision"]) + for pinned in source["files"] + require_keys(pinned, ["file", "sha256"]) + @test pinned["file"] isa String + @test pinned["sha256"] isa String + @test safe_relative(pinned["file"]) + @test occursin(SHA256_PATTERN, pinned["sha256"]) + end + end + @test length(sources) == length(unique(item["id"] for item in sources)) + return +end + +function validate_manifest_toolchains(toolchains) + required = ["id", "status", "version", "platform", "scope", "artifacts"] + allowed = vcat(required, ["distribution_url", "tree_policy"]) + for toolchain in toolchains + require_keys(toolchain, allowed, required) + @test all(field -> toolchain[field] isa String, + ("id", "status", "version", "platform", "scope")) + @test toolchain["status"] in ("verified", "planned") + @test !isempty(toolchain["artifacts"]) + names = String[] + for artifact in toolchain["artifacts"] + require_keys(artifact, ["name", "sha256"]) + @test artifact["name"] isa String + @test artifact["sha256"] isa String + @test occursin(SHA256_PATTERN, artifact["sha256"]) + push!(names, artifact["name"]) + end + @test length(names) == length(unique(names)) + end + @test length(toolchains) == length(unique(item["id"] for item in toolchains)) + validator = only(filter(item -> item["id"] == "jsonschema-validator", + toolchains)) + @test validator["distribution_url"] == CPYTHON_314_DISTRIBUTION_URL + @test validator["tree_policy"] == VALIDATOR_CLEAN_TREE_POLICY + return +end + +function validate_manifest_models(model_files) + for model_file in model_files + require_keys(model_file, ["file", "sha256"]) + @test model_file["file"] isa String + @test model_file["sha256"] isa String + @test safe_relative(model_file["file"]) + @test occursin(SHA256_PATTERN, model_file["sha256"]) + end + return +end + +function validate_frozen_evidence(evidence) + allowed = ["id", "status", "authority", + "toolchain_sha256", "file", "format", "storage", "schema_file", + "schema_sha256", "fixture_manifest_file", "case_count", + "record_count", "sha256", "upstream_evidence", "scope"] + require_keys(evidence, allowed, setdiff(allowed, ["upstream_evidence"])) + @test all(field -> evidence[field] isa String, ( + "id", "status", "authority", "toolchain_sha256", "file", + "format", "storage", "schema_file", "schema_sha256", + "fixture_manifest_file", "sha256", "scope")) + @test occursin(CASE_PATTERN, evidence["id"]) + @test evidence["status"] == "verified" + @test occursin(SHA256_PATTERN, evidence["toolchain_sha256"]) + @test safe_evidence_file(evidence["file"]) + @test evidence["format"] in ("raw-jsonl", "normalized-jsonl") + @test evidence["storage"] in ("checked-in", "gate-generated") + @test safe_relative(evidence["schema_file"]) + @test safe_relative(evidence["fixture_manifest_file"]) + @test occursin(SHA256_PATTERN, evidence["schema_sha256"]) + @test occursin(SHA256_PATTERN, evidence["sha256"]) + @test evidence["case_count"] isa Int64 + @test evidence["record_count"] isa Int64 + @test evidence["case_count"] > 0 + @test evidence["record_count"] >= evidence["case_count"] + @test !isempty(evidence["scope"]) + upstream = get(evidence, "upstream_evidence", String[]) + @test all(item -> item isa String && occursin(CASE_PATTERN, item), upstream) + @test length(upstream) == length(unique(upstream)) + @test evidence["id"] ∉ upstream + return evidence["id"], evidence["file"] +end + +function validate_planned_evidence(evidence, manifest) + allowed = ["id", "status", "authority", "file", "format", + "schema_file", "fixture_manifest_file", "upstream_evidence", "scope"] + require_keys(evidence, allowed, setdiff(allowed, ["upstream_evidence"])) + @test all(field -> evidence[field] isa String, ( + "id", "status", "authority", "file", "format", "schema_file", + "fixture_manifest_file", "scope")) + @test occursin(CASE_PATTERN, evidence["id"]) + @test evidence["status"] == "planned" + @test safe_evidence_file(evidence["file"]) + @test evidence["format"] == "normalized-jsonl" + @test safe_relative(evidence["schema_file"]) + @test safe_relative(evidence["fixture_manifest_file"]) + @test evidence["schema_file"] == manifest["evidence_schema_file"] + @test evidence["fixture_manifest_file"] == manifest["fixture_manifest_file"] + @test !isempty(evidence["scope"]) + upstream = get(evidence, "upstream_evidence", String[]) + @test all(item -> item isa String && occursin(CASE_PATTERN, item), upstream) + @test length(upstream) == length(unique(upstream)) + @test evidence["id"] ∉ upstream + return evidence["id"], evidence["file"] +end + +function validate_manifest_evidence(manifest) + evidence_ids = String[] + evidence_files = String[] + for evidence in manifest["frozen_evidence"] + id, file = validate_frozen_evidence(evidence) + push!(evidence_ids, id) + push!(evidence_files, file) + end + for evidence in manifest["planned_evidence"] + id, file = validate_planned_evidence(evidence, manifest) + push!(evidence_ids, id) + push!(evidence_files, file) + end + @test length(evidence_ids) == length(unique(evidence_ids)) + @test length(evidence_files) == length(unique(evidence_files)) + known = Set(evidence_ids) + for evidence in vcat(manifest["frozen_evidence"], + manifest["planned_evidence"]) + @test all(item -> item in known, + get(evidence, "upstream_evidence", String[])) + end + return +end + +function validate_manifest(manifest) + validate_manifest_header(manifest) + validate_manifest_sources(manifest["source"]) + validate_manifest_toolchains(manifest["toolchain"]) + validate_manifest_models(manifest["frozen_model"]) + validate_manifest_evidence(manifest) + return +end + +function validate_capability_catalog(capabilities, manifest) + require_keys(capabilities, ["matrix_version", "plan_sha256", + "fixture_manifest", "unsupported_is_pass", "statuses", "capability", + "authority"]) + @test capabilities["matrix_version"] == 2 + @test capabilities["unsupported_is_pass"] === false + @test capabilities["plan_sha256"] isa String + @test capabilities["plan_sha256"] == manifest["plan_sha256"] + @test capabilities["fixture_manifest"] isa String + @test capabilities["fixture_manifest"] == + basename(manifest["fixture_manifest_file"]) + @test capabilities["statuses"] == + ["verified", "planned", "unsupported", "not_assessed"] + @test all(value -> value isa String, capabilities["statuses"]) + capability_ids = Set{String}() + for capability in capabilities["capability"] + require_keys(capability, ["id", "kind"]) + @test capability["id"] isa String + @test capability["kind"] isa String + @test occursin(CASE_PATTERN, capability["id"]) + @test capability["kind"] in ("wire", "semantic", "runtime", "compatibility") + @test capability["id"] ∉ capability_ids + push!(capability_ids, capability["id"]) + end + return capability_ids +end + +function capability_case_context(fixtures, model_cases) + case_capabilities = Dict(item["id"] => Set(item["capabilities"]) + for item in vcat(fixtures["fixture"], fixtures["generated_case"])) + for item in model_cases["case_groups"] + @test !haskey(case_capabilities, item["id"]) + case_capabilities[item["id"]] = Set(item["capabilities"]) + end + return case_capabilities, Set(keys(case_capabilities)) +end + +function validate_authority_claim(claim, capabilities, capability_ids, known_cases, + case_capabilities, seen, claim_pairs) + require_keys(claim, ["capability", "status", "cases", "scope"]) + @test all(field -> claim[field] isa String, + ("capability", "status", "scope")) + @test all(case_id -> case_id isa String, claim["cases"]) + @test claim["capability"] in capability_ids + @test claim["status"] in capabilities["statuses"] + @test !isempty(claim["cases"]) + @test !isempty(claim["scope"]) + @test length(claim["cases"]) == length(unique(claim["cases"])) + @test all(case_id -> case_id in known_cases, claim["cases"]) + for case_id in claim["cases"] + @test claim["capability"] in case_capabilities[case_id] + key = (claim["capability"], case_id) + @test !haskey(seen, key) + seen[key] = claim["status"] + push!(claim_pairs, (case_id, claim["capability"])) + end + return +end + +function validate_authority(authority, capabilities, capability_ids, known_cases, + case_capabilities, artifact_hashes, revisions, + authority_ids, claim_pairs) + require_keys(authority, ["id", "kind", "version", "revision", "platforms", + "toolchain_sha256", "claim"]) + @test all(field -> authority[field] isa String, + ("id", "kind", "version", "revision")) + @test all(value -> value isa String, authority["platforms"]) + @test all(value -> value isa String, authority["toolchain_sha256"]) + @test authority["id"] ∉ authority_ids + push!(authority_ids, authority["id"]) + @test !isempty(authority["kind"]) + @test !isempty(authority["version"]) + @test !isempty(authority["revision"]) + @test authority["revision"] in revisions + @test !isempty(authority["platforms"]) + @test length(authority["toolchain_sha256"]) == + length(unique(authority["toolchain_sha256"])) + @test all(digest -> occursin(SHA256_PATTERN, digest), + authority["toolchain_sha256"]) + @test all(digest -> digest in artifact_hashes, + authority["toolchain_sha256"]) + seen = Dict{Tuple{String,String},String}() + for claim in authority["claim"] + validate_authority_claim(claim, capabilities, capability_ids, known_cases, + case_capabilities, seen, claim_pairs) + end + return +end + +function validate_authorities(capabilities, fixtures, model_cases, manifest, + capability_ids) + case_capabilities, known_cases = + capability_case_context(fixtures, model_cases) + authority_ids = Set{String}() + claim_pairs = Set{Tuple{String,String}}() + artifact_hashes = Set(artifact["sha256"] for toolchain in manifest["toolchain"] + for artifact in toolchain["artifacts"]) + revisions = Set(source["revision"] for source in manifest["source"]) + union!(revisions, Set(model["sha256"] for model in manifest["frozen_model"])) + push!(revisions, manifest["model_producer_descriptor_sha256"]) + push!(revisions, manifest["parquet_jl_source_composite_sha256"]) + push!(revisions, "preproduction") + for authority in capabilities["authority"] + validate_authority(authority, capabilities, capability_ids, known_cases, + case_capabilities, artifact_hashes, revisions, + authority_ids, claim_pairs) + end + @test length(authority_ids) == length(capabilities["authority"]) + return authority_ids, claim_pairs +end + +function validate_evidence_authorities(capabilities, fixtures, manifest, + authority_ids) + for evidence in manifest["frozen_evidence"] + @test evidence["authority"] in authority_ids + authority = only(filter(item -> item["id"] == evidence["authority"], + capabilities["authority"])) + @test evidence["toolchain_sha256"] in authority["toolchain_sha256"] + owning_toolchains = filter(toolchain -> any(artifact -> + artifact["sha256"] == evidence["toolchain_sha256"], + toolchain["artifacts"]), manifest["toolchain"]) + @test length(owning_toolchains) == 1 + @test only(owning_toolchains)["status"] == "verified" + @test evidence["case_count"] <= + length(fixtures["fixture"]) + length(fixtures["generated_case"]) + @test evidence["record_count"] <= + manifest["evidence_limits"]["max_records_per_input"] + if evidence["format"] == "normalized-jsonl" + @test evidence["schema_file"] == manifest["evidence_schema_file"] + @test evidence["schema_sha256"] == manifest["evidence_schema_sha256"] + end + end + raw_evidence = only(filter(item -> item["id"] == "raw-java-apache-corpus", + manifest["frozen_evidence"])) + @test raw_evidence["case_count"] == length(fixtures["fixture"]) + @test all(evidence -> evidence["authority"] in authority_ids, + manifest["planned_evidence"]) + return +end + +function validate_fixture_claim_coverage(fixtures, claim_pairs) + for case in vcat(fixtures["fixture"], fixtures["generated_case"]) + @test all(capability -> (case["id"], capability) in claim_pairs, + case["capabilities"]) + end + return +end + +function validate_model_claim_coverage(model_cases, claim_pairs) + for case in model_cases["case_groups"] + @test all(capability -> (case["id"], capability) in claim_pairs, + case["capabilities"]) + end + return +end + +function validate_raw_claim_coverage(capabilities, fixtures) + raw_authority = only(filter(item -> item["id"] == "n6-raw-java", + capabilities["authority"])) + raw_claim_pairs = Set((case_id, claim["capability"]) + for claim in raw_authority["claim"] for case_id in claim["cases"]) + for fixture in fixtures["fixture"] + for capability in fixture["capabilities"] + startswith(capability, "wire.") || continue + @test (fixture["id"], capability) in raw_claim_pairs + end + end + for case_id in ("julia-writer-type-order", "julia-writer-ieee-order", + "julia-writer-nested-row-groups") + generated = only(filter(item -> item["id"] == case_id, + fixtures["generated_case"])) + @test "wire.statistics.exactness" in generated["capabilities"] + @test (case_id, "wire.statistics.exactness") in raw_claim_pairs + end + return +end + +function validate_capabilities(capabilities, fixtures, model_cases, manifest) + capability_ids = validate_capability_catalog(capabilities, manifest) + authority_ids, claim_pairs = validate_authorities(capabilities, fixtures, + model_cases, manifest, capability_ids) + validate_evidence_authorities(capabilities, fixtures, manifest, authority_ids) + validate_fixture_claim_coverage(fixtures, claim_pairs) + validate_model_claim_coverage(model_cases, claim_pairs) + validate_raw_claim_coverage(capabilities, fixtures) + return capability_ids +end + +function validate_fixture_manifest_header(fixtures, manifest) + require_keys(fixtures, ["manifest_version", "authority", "source_revision", + "checksum_manifest", "status_values", "output_identity_statuses", + "generation_contract_version", "default_digest_contract", + "digest_contracts", "fixture", "generated_case"]) + @test fixtures["manifest_version"] == 1 + @test all(field -> fixtures[field] isa String, ( + "authority", "source_revision", "checksum_manifest", + "default_digest_contract")) + testing_source = only(filter(item -> item["id"] == "parquet-testing", + manifest["source"])) + @test fixtures["authority"] == "parquet-testing" + @test fixtures["source_revision"] == testing_source["revision"] + @test fixtures["checksum_manifest"] == + basename(manifest["corpus_manifest_file"]) + @test fixtures["status_values"] == ["verified", "planned", "unsupported"] + @test fixtures["output_identity_statuses"] == ["planned", "verified"] + @test all(value -> value isa String, fixtures["status_values"]) + @test all(value -> value isa String, fixtures["output_identity_statuses"]) + @test fixtures["generation_contract_version"] == 1 + @test fixtures["default_digest_contract"] == + "n6-capability-result-sha256-v1" + @test fixtures["digest_contracts"] == ["n6-capability-result-sha256-v1", + "n6-no-pruning-trace-sha256-v1"] + @test all(value -> value isa String, fixtures["digest_contracts"]) + return +end + +function validate_apache_fixture(fixture, fixtures, capability_ids, ids, files) + require_keys(fixture, ["id", "status", "source_kind", "authority", "file", + "sha256", "size", "source_revision", "row_group_count", "leaf_count", + "normalized_record_count", "capabilities", "expected_unsupported"]) + @test all(field -> fixture[field] isa String, ( + "id", "status", "source_kind", "authority", "file", "sha256", + "source_revision")) + @test all(value -> value isa String, fixture["capabilities"]) + @test all(value -> value isa String, fixture["expected_unsupported"]) + @test occursin(CASE_PATTERN, fixture["id"]) + @test fixture["id"] ∉ ids + push!(ids, fixture["id"]) + @test fixture["status"] in fixtures["status_values"] + @test fixture["source_kind"] == "apache-corpus" + @test fixture["authority"] == fixtures["authority"] + @test fixture["source_revision"] == fixtures["source_revision"] + @test safe_relative(fixture["file"]) + @test fixture["file"] ∉ keys(files) + files[fixture["file"]] = fixture["sha256"] + @test all(field -> fixture[field] isa Int64, + ("size", "row_group_count", "leaf_count", "normalized_record_count")) + @test fixture["size"] >= 12 + @test fixture["row_group_count"] >= 0 + @test fixture["leaf_count"] >= 0 + @test fixture["normalized_record_count"] == + 1 + fixture["row_group_count"] * fixture["leaf_count"] + @test !isempty(fixture["capabilities"]) + @test length(fixture["capabilities"]) == length(unique(fixture["capabilities"])) + @test all(capability -> capability in capability_ids, fixture["capabilities"]) + @test all(capability -> capability in fixture["capabilities"], + fixture["expected_unsupported"]) + return +end + +function validate_apache_fixtures(fixtures, capability_ids) + ids = Set{String}() + files = Dict{String,String}() + for fixture in fixtures["fixture"] + validate_apache_fixture(fixture, fixtures, capability_ids, ids, files) + end + @test length(ids) == 20 + return ids, files +end + +function validate_generated_output(generated, fixtures, ids, output_files) + allowed = ["id", "status", "source_kind", "authority", "output_file", + "output_identity_status", "output_sha256", "output_size", + "generator_profile", "generator_seed", "variant_id", "mutation", + "comparison_group", "digest_contract", "row_group_count", "leaf_count", + "normalized_record_count", "capabilities", "expected_unsupported", + "description"] + required = setdiff(allowed, ["output_sha256", "output_size"]) + require_keys(generated, allowed, required) + @test all(field -> generated[field] isa String, ( + "id", "status", "source_kind", "authority", "output_file", + "output_identity_status", "generator_profile", "variant_id", + "comparison_group", "digest_contract", "description")) + @test all(value -> value isa String, generated["capabilities"]) + @test all(value -> value isa String, generated["expected_unsupported"]) + @test occursin(CASE_PATTERN, generated["id"]) + @test generated["id"] ∉ ids + push!(ids, generated["id"]) + @test generated["status"] == "verified" + @test generated["source_kind"] in + ("julia-writer-generated", "metadata-mutation-generated") + @test generated["authority"] == "parquet-jl" + @test safe_generated_output(generated["output_file"]) + @test generated["output_file"] ∉ output_files + push!(output_files, generated["output_file"]) + identity_status = generated["output_identity_status"] + @test identity_status in fixtures["output_identity_statuses"] + if identity_status == "planned" + @test !haskey(generated, "output_sha256") + @test !haskey(generated, "output_size") + else + @test generated["output_sha256"] isa String + @test generated["output_size"] isa Int64 + @test occursin(SHA256_PATTERN, generated["output_sha256"]) + @test generated["output_size"] >= 12 + end + return +end + +function validate_generated_profile(generated, fixtures, generation_identities) + @test occursin(CASE_PATTERN, generated["generator_profile"]) + @test all(field -> generated[field] isa Int64, + ("generator_seed", "row_group_count", "leaf_count", + "normalized_record_count")) + @test generated["generator_seed"] >= 0 + @test occursin(CASE_PATTERN, generated["variant_id"]) + identity = (generated["generator_profile"], + generated["generator_seed"], generated["variant_id"]) + @test identity ∉ generation_identities + push!(generation_identities, identity) + @test generated["comparison_group"] == "" || + occursin(CASE_PATTERN, generated["comparison_group"]) + @test generated["digest_contract"] in fixtures["digest_contracts"] + return +end + +function validate_statistics_state_mutation(mutation) + state = mutation["statistics_state"] + @test state isa String + if state == "absent" + require_keys(mutation, ["kind", "statistics_state"]) + elseif state in ("trusted", "producer-untrusted") + require_keys(mutation, ["kind", "statistics_state", "created_by"]) + @test mutation["created_by"] isa String + @test !isempty(mutation["created_by"]) + elseif state == "oversized" + require_keys(mutation, ["kind", "statistics_state", "bound_bytes"]) + @test mutation["bound_bytes"] isa Int64 + @test mutation["bound_bytes"] == 4097 + elseif state == "semantically-unusable" + require_keys(mutation, ["kind", "statistics_state", "field", "value_hex"]) + @test mutation["field"] isa String + @test mutation["value_hex"] isa String + @test mutation["field"] == "min_value" + @test mutation["value_hex"] == "000000" + else + @test false + end + return +end + +function validate_generated_mutation(mutation) + kind = mutation["kind"] + @test kind isa String + if kind == "none" + require_keys(mutation, ["kind"]) + elseif kind == "created-by" + require_keys(mutation, ["kind", "created_by"]) + @test mutation["created_by"] isa String + @test !isempty(mutation["created_by"]) + elseif kind == "statistics-state" + validate_statistics_state_mutation(mutation) + else + @test false + end + return +end + +function validate_generated_topology(generated, capability_ids) + @test generated["row_group_count"] >= 0 + @test generated["leaf_count"] >= 0 + @test generated["normalized_record_count"] == + 1 + generated["row_group_count"] * generated["leaf_count"] + @test !isempty(generated["capabilities"]) + @test length(generated["capabilities"]) == + length(unique(generated["capabilities"])) + @test all(capability -> capability in capability_ids, + generated["capabilities"]) + @test all(capability -> capability in generated["capabilities"], + generated["expected_unsupported"]) + return +end + +function validate_generated_fixtures(fixtures, capability_ids, ids, files) + output_files = Set(keys(files)) + generation_identities = Set{Tuple{String,Int64,String}}() + for generated in fixtures["generated_case"] + validate_generated_output(generated, fixtures, ids, output_files) + validate_generated_profile(generated, fixtures, generation_identities) + validate_generated_mutation(generated["mutation"]) + validate_generated_topology(generated, capability_ids) + end + @test length(ids) == 32 + return +end + +function validate_no_pruning_fixtures(fixtures) + no_pruning = filter(item -> + item["comparison_group"] == "julia-reader-no-pruning-v1", + fixtures["generated_case"]) + @test Set(item["variant_id"] for item in no_pruning) == + Set(["absent", "trusted", "producer-untrusted", "oversized", + "semantically-unusable"]) + @test all(item -> item["generator_profile"] == "reader-no-pruning-v1" && + item["generator_seed"] == 6008 && item["digest_contract"] == + "n6-no-pruning-trace-sha256-v1", no_pruning) + return +end + +function validate_fixture_evidence_limits(fixtures, model_cases, manifest) + maximum_records = 1 + sum(item["normalized_record_count"] + + length(item["capabilities"]) for item in + vcat(fixtures["fixture"], fixtures["generated_case"])) + maximum_records += sum(length(item["capabilities"]) + for item in model_cases["case_groups"]) + @test manifest["evidence_limits"]["max_records_per_input"] == maximum_records + @test manifest["evidence_limits"]["max_records_total"] == + maximum_records * manifest["evidence_limits"]["max_inputs"] + return +end + +function validate_corpus_manifest(corpus_lines, files) + corpus = Dict{String,String}() + corpus_paths = String[] + for (line_number, line) in enumerate(corpus_lines) + match_result = match(r"^([0-9a-f]{64}) ([A-Za-z0-9._+@=/\-]+)$", line) + match_result === nothing && error("invalid corpus line $line_number") + digest, relative = match_result.captures + safe_relative(relative) || error("unsafe corpus path: $relative") + haskey(corpus, relative) && error("duplicate corpus path: $relative") + corpus[relative] = digest + push!(corpus_paths, relative) + end + @test corpus_paths == sort(corpus_paths) + @test corpus == files + return +end + +function validate_fixtures(fixtures, model_cases, corpus_lines, capability_ids, + manifest) + validate_fixture_manifest_header(fixtures, manifest) + ids, files = validate_apache_fixtures(fixtures, capability_ids) + validate_generated_fixtures(fixtures, capability_ids, ids, files) + validate_no_pruning_fixtures(fixtures) + validate_fixture_evidence_limits(fixtures, model_cases, manifest) + validate_corpus_manifest(corpus_lines, files) + return +end + +function validate_python_descriptors(manifest, capabilities, snapshots) + python_source = only(filter(item -> item["id"] == "cpython-3.12", + manifest["source"])) + python_toolchain = only(filter(item -> item["id"] == "python-interop", + manifest["toolchain"])) + artifact_map = Dict(item["name"] => item["sha256"] + for item in python_toolchain["artifacts"]) + descriptors = Dict( + "toolchains/pyarrow.toml" => ( + "pyarrow", "pyarrow-cp312-macos-15-arm64", + "pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl"), + "toolchains/duckdb.toml" => ( + "duckdb", "duckdb-cp312-macos-15-arm64", + "duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl")) + for (relative, (authority_id, descriptor_id, wheel_name)) in descriptors + repository_relative = "test/conformance/n6/" * relative + snapshot = control_snapshot(snapshots, repository_relative) + @test snapshot.sha256 == artifact_map[relative] + descriptor = parse_toml_snapshot(snapshot) + allowed = ["descriptor_version", "id", "status", "authority", "platform", + "python_version", "python_source_revision", + "python_distribution_url", "python_distribution_sha256", + "python_tree_policy", "python_executable_sha256", + "python_tree_sha256", "wheels", + "harness_status", "harness_file", "harness_sha256", + "support_files", "test_file", "test_sha256"] + require_keys(descriptor, allowed) + @test all(field -> descriptor[field] isa String, ( + "id", "status", "authority", "platform", "python_version", + "python_source_revision", "python_distribution_url", + "python_distribution_sha256", "python_tree_policy", + "python_executable_sha256", "python_tree_sha256", "harness_status", + "harness_file", "harness_sha256", "test_file", "test_sha256")) + @test descriptor["descriptor_version"] == 1 + @test descriptor["id"] == descriptor_id + @test descriptor["status"] == python_toolchain["status"] + @test descriptor["authority"] == authority_id + @test descriptor["platform"] == "macos-15-arm64" + @test descriptor["python_version"] == python_source["version"] + @test descriptor["python_source_revision"] == python_source["revision"] + @test descriptor["python_distribution_url"] == + CPYTHON_312_DISTRIBUTION_URL + @test descriptor["python_distribution_sha256"] == + artifact_map["cpython-distribution-archive"] + @test descriptor["python_tree_policy"] == CPYTHON_CLEAN_TREE_POLICY + @test descriptor["python_executable_sha256"] == + artifact_map["cpython-executable"] + @test descriptor["python_tree_sha256"] == + artifact_map["cpython-clean-tree-sha256-v1"] + @test descriptor["wheels"] == + [Dict("name" => wheel_name, "sha256" => artifact_map[wheel_name])] + authority = only(filter(item -> item["id"] == authority_id, + capabilities["authority"])) + @test authority["toolchain_sha256"] == [artifact_map[relative]] + @test safe_relative(descriptor["harness_file"]) + @test descriptor["harness_status"] == descriptor["status"] + @test occursin(SHA256_PATTERN, descriptor["harness_sha256"]) + @test control_snapshot(snapshots, + descriptor["harness_file"]).sha256 == descriptor["harness_sha256"] + @test safe_relative(descriptor["test_file"]) + @test occursin(SHA256_PATTERN, descriptor["test_sha256"]) + @test control_snapshot(snapshots, + descriptor["test_file"]).sha256 == descriptor["test_sha256"] + @test descriptor["support_files"] isa Vector + @test !isempty(descriptor["support_files"]) + support_paths = String[] + for support in descriptor["support_files"] + require_keys(support, ["file", "sha256"]) + @test support["file"] isa String + @test support["sha256"] isa String + @test safe_relative(support["file"]) + @test occursin(SHA256_PATTERN, support["sha256"]) + @test control_snapshot(snapshots, + support["file"]).sha256 == support["sha256"] + push!(support_paths, support["file"]) + end + @test length(support_paths) == length(unique(support_paths)) + end + return +end + +function validate_arrow_descriptor(manifest, capabilities, snapshots) + relative = "oracles/arrow-rs/toolchain.toml" + snapshot = control_snapshot(snapshots, "test/conformance/n6/" * relative) + toolchain = only(filter(item -> item["id"] == "rust-arrow-rs", + manifest["toolchain"])) + artifact = only(filter(item -> item["name"] == relative, + toolchain["artifacts"])) + @test snapshot.sha256 == artifact["sha256"] + descriptor = parse_toml_snapshot(snapshot) + require_keys(descriptor, ["descriptor_version", "status", "producer", + "producer_version", "source_revision", "image_reference", "image_id", + "image_platform", "binary_path", "binary_sha256", "platform", + "metadata_binary_file", "metadata_binary_sha256", + "metadata_binary_size", + "python_version", "python_distribution_url", + "python_distribution_sha256", "python_tree_policy", + "python_executable_sha256", "python_tree_sha256", + "rust_toolchain", "rustc", "cargo", "source", "wrapper"]) + @test descriptor["descriptor_version"] == 1 + @test descriptor["status"] == toolchain["status"] + @test descriptor["producer"] == "arrow-rs" + authority = only(filter(item -> item["id"] == "arrow-rs", + capabilities["authority"])) + source = only(filter(item -> item["id"] == "arrow-rs", manifest["source"])) + @test descriptor["producer_version"] == authority["version"] == + source["version"] + @test descriptor["source_revision"] == authority["revision"] == + source["revision"] + @test descriptor["platform"] == "macos-15-arm64" + @test descriptor["image_platform"] == "linux/amd64" + @test startswith(descriptor["image_id"], "sha256:") + @test occursin(SHA256_PATTERN, descriptor["image_id"][8:end]) + @test startswith(descriptor["binary_path"], "/") + @test occursin(SHA256_PATTERN, descriptor["binary_sha256"]) + @test descriptor["metadata_binary_file"] == + "test/conformance/n6/oracles/arrow-rs/build/" * + "parquet-jl-n6-arrow-rs-metadata" + @test descriptor["metadata_binary_sha256"] == + toolchain_artifact(manifest, "rust-arrow-rs", + "parquet-jl-n6-arrow-rs-metadata") + @test descriptor["metadata_binary_size"] == 710064 + python_source = only(filter(item -> item["id"] == "cpython-3.12", + manifest["source"])) + python_toolchain = only(filter(item -> item["id"] == "python-interop", + manifest["toolchain"])) + python_artifacts = Dict(item["name"] => item["sha256"] + for item in python_toolchain["artifacts"]) + @test descriptor["python_version"] == python_source["version"] + @test descriptor["python_distribution_url"] == CPYTHON_312_DISTRIBUTION_URL + @test descriptor["python_distribution_sha256"] == + python_artifacts["cpython-distribution-archive"] + @test descriptor["python_tree_policy"] == CPYTHON_CLEAN_TREE_POLICY + @test descriptor["python_executable_sha256"] == + python_artifacts["cpython-executable"] + @test descriptor["python_tree_sha256"] == + python_artifacts["cpython-clean-tree-sha256-v1"] + source_paths = String[] + for entry in descriptor["source"] + require_keys(entry, ["path", "sha256"]) + @test entry["path"] isa String + @test entry["sha256"] isa String + @test startswith(entry["path"], "/opt/bootstrap/arrow-rs/") + @test occursin(SHA256_PATTERN, entry["sha256"]) + push!(source_paths, entry["path"]) + end + @test length(source_paths) == 12 + @test length(source_paths) == length(unique(source_paths)) + wrapper_paths = String[] + for entry in descriptor["wrapper"] + require_keys(entry, ["path", "sha256"]) + @test entry["path"] isa String + @test entry["sha256"] isa String + @test safe_relative(entry["path"]) + @test occursin(SHA256_PATTERN, entry["sha256"]) + @test control_snapshot(snapshots, + entry["path"]).sha256 == entry["sha256"] + push!(wrapper_paths, entry["path"]) + end + @test Set(wrapper_paths) == Set([ + "test/conformance/n6/harnesses/common.py", + "test/conformance/n6/oracles/arrow-rs/build.sh", + "test/conformance/n6/oracles/arrow-rs/run.py", + "test/conformance/n6/oracles/arrow-rs/run.sh", + "test/conformance/n6/oracles/arrow-rs/runtests.py", + "test/conformance/n6/oracles/arrow-rs/check.sh", + "test/conformance/n6/oracles/arrow-rs/metadata/Cargo.toml", + "test/conformance/n6/oracles/arrow-rs/metadata/src/main.rs", + ]) + @test authority["toolchain_sha256"] == [artifact["sha256"]] + return +end + +function validate_parquet_java_descriptor(manifest, capabilities, snapshots) + relative = "oracles/parquet-java/toolchain.toml" + snapshot = control_snapshot(snapshots, "test/conformance/n6/" * relative) + toolchain = only(filter(item -> item["id"] == "parquet-java-interop", + manifest["toolchain"])) + artifact_map = Dict(item["name"] => item["sha256"] + for item in toolchain["artifacts"]) + @test snapshot.sha256 == artifact_map[relative] + descriptor = parse_toml_snapshot(snapshot) + require_keys(descriptor, [ + "descriptor_version", "status", "producer", "producer_version", + "source_revision", "platform", "python_version", + "python_distribution_url", "python_distribution_sha256", + "python_tree_policy", "python_executable_sha256", "python_tree_sha256", + "java_vendor", + "java_version", "java_executable_sha256", "javac_executable_sha256", + "java_release_sha256", "jdk_tree_sha256", "hadoop_version", + "harness_main", "harness_jar_file", "harness_jar_sha256", + "harness_jar_size", "artifact", "source", "wrapper", + ]) + @test descriptor["descriptor_version"] == 1 + @test descriptor["status"] == toolchain["status"] + @test descriptor["producer"] == "parquet-java" + authority = only(filter(item -> item["id"] == "parquet-java", + capabilities["authority"])) + source = only(filter(item -> item["id"] == "parquet-java", + manifest["source"])) + @test descriptor["producer_version"] == authority["version"] == + source["version"] + @test descriptor["source_revision"] == authority["revision"] == + source["revision"] + @test descriptor["platform"] == toolchain["platform"] == + "macos-15-arm64" + python_source = only(filter(item -> item["id"] == "cpython-3.12", + manifest["source"])) + @test descriptor["python_version"] == python_source["version"] + @test descriptor["python_distribution_url"] == CPYTHON_312_DISTRIBUTION_URL + @test descriptor["python_distribution_sha256"] == + artifact_map["cpython-distribution-archive"] + @test descriptor["python_tree_policy"] == CPYTHON_CLEAN_TREE_POLICY + @test descriptor["python_executable_sha256"] == + artifact_map["cpython-executable"] + @test descriptor["python_tree_sha256"] == + artifact_map["cpython-clean-tree-sha256-v1"] + @test descriptor["java_vendor"] == "Eclipse-Adoptium-Temurin" + @test descriptor["java_version"] == "21.0.8+9" + @test descriptor["java_executable_sha256"] == + artifact_map["temurin-java"] + @test descriptor["javac_executable_sha256"] == + artifact_map["temurin-javac"] + @test descriptor["java_release_sha256"] == + artifact_map["temurin-release"] + @test descriptor["jdk_tree_sha256"] == + artifact_map["temurin-tree-sha256-v1"] + @test descriptor["hadoop_version"] == "3.3.0" + @test descriptor["harness_main"] == + "org.julialang.parquet.n6.java.AuditMain" + @test descriptor["harness_jar_file"] == + "test/conformance/n6/oracles/parquet-java/build/artifacts/" * + "parquet-java-n6-harness.jar" + @test descriptor["harness_jar_sha256"] == + artifact_map["parquet-java-n6-harness.jar"] + @test descriptor["harness_jar_size"] == 9408 + expected_artifacts = Dict( + "parquet-cli-runtime" => ( + "parquet-cli-1.17.1-runtime.jar", 50072122), + "hadoop-client-api" => ("hadoop-client-api-3.3.0.jar", 19207034), + "hadoop-client-runtime" => ( + "hadoop-client-runtime-3.3.0.jar", 27255121), + ) + @test Set(item["id"] for item in descriptor["artifact"]) == + Set(keys(expected_artifacts)) + for item in descriptor["artifact"] + require_keys(item, ["id", "file", "url", "sha256", "size"]) + name, size = expected_artifacts[item["id"]] + @test item["file"] == + "test/conformance/n6/oracles/parquet-java/build/artifacts/$name" + @test item["url"] == + "https://repo.maven.apache.org/maven2/" * + (item["id"] == "parquet-cli-runtime" ? + "org/apache/parquet/parquet-cli/1.17.1/$name" : + "org/apache/hadoop/" * + replace(name, r"-[0-9].*" => "") * "/3.3.0/$name") + @test item["sha256"] == artifact_map[name] + @test item["size"] == size + end + expected_sources = Set([ + "parquet-common/src/main/java/org/apache/parquet/VersionParser.java", + "parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java", + "parquet-common/src/main/java/org/apache/parquet/io/LocalInputFile.java", + "parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java", + "parquet-column/src/main/java/org/apache/parquet/example/data/Group.java", + "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java", + "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java", + "parquet-hadoop/src/main/java/org/apache/parquet/hadoop/example/GroupReadSupport.java", + "parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java", + ]) + source_map = Dict{String,String}() + for item in descriptor["source"] + require_keys(item, ["file", "sha256"]) + @test safe_relative(item["file"]) + @test occursin(SHA256_PATTERN, item["sha256"]) + @test !haskey(source_map, item["file"]) + source_map[item["file"]] = item["sha256"] + end + @test Set(keys(source_map)) == expected_sources + for item in source["files"] + @test get(source_map, item["file"], nothing) == item["sha256"] + end + expected_wrappers = Set([ + "test/conformance/n6/harnesses/common.py", + "test/conformance/n6/oracles/parquet-java/build.sh", + "test/conformance/n6/oracles/parquet-java/check.sh", + "test/conformance/n6/oracles/parquet-java/run.py", + "test/conformance/n6/oracles/parquet-java/run.sh", + "test/conformance/n6/oracles/parquet-java/runtests.py", + "test/conformance/n6/oracles/parquet-java/src/org/julialang/parquet/n6/java/AuditMain.java", + ]) + wrapper_paths = Set{String}() + for item in descriptor["wrapper"] + require_keys(item, ["path", "sha256"]) + @test safe_relative(item["path"]) + @test occursin(SHA256_PATTERN, item["sha256"]) + @test item["path"] ∉ wrapper_paths + @test control_snapshot(snapshots, + item["path"]).sha256 == item["sha256"] + push!(wrapper_paths, item["path"]) + end + @test wrapper_paths == expected_wrappers + @test authority["toolchain_sha256"] == [artifact_map[relative]] + return +end + +function validate_static_files(manifest, capabilities, fixtures, snapshots) + for (file_key, hash_key) in ( + ("plan_file", "plan_sha256"), + ("capabilities_file", "capabilities_sha256"), + ("fixture_manifest_file", "fixture_manifest_sha256"), + ("corpus_manifest_file", "corpus_manifest_sha256"), + ("evidence_schema_file", "evidence_schema_sha256"), + ("artifact_manifest_file", "artifact_manifest_sha256"), + ("model_producer_descriptor_file", + "model_producer_descriptor_sha256"), + ("parquet_jl_producer_descriptor_file", + "parquet_jl_producer_descriptor_sha256")) + snapshot = control_snapshot(snapshots, manifest[file_key]) + @test snapshot.sha256 == manifest[hash_key] + end + for model_file in manifest["frozen_model"] + @test control_snapshot(snapshots, + model_file["file"]).sha256 == model_file["sha256"] + end + descriptor = parse_toml_snapshot(control_snapshot(snapshots, + manifest["model_producer_descriptor_file"])) + require_keys(descriptor, ["descriptor_version", "producer", "file"]) + @test descriptor["descriptor_version"] == 1 + @test descriptor["producer"] == "n6-independent-model" + expected_producer_files = Set([ + "test/conformance/n6/model/N6StatisticsModel.jl", + "test/conformance/n6/model/README.md", + "test/conformance/n6/model/cases.toml", + "test/conformance/n6/model/runtests.jl", + "test/conformance/n6/normalizers/common.py", + "test/conformance/n6/normalizers/model_bridge.jl", + "test/conformance/n6/normalizers/normalize_model.py", + "test/conformance/n6/normalizers/normalize_raw.py", + ]) + producer_paths = Set{String}() + for item in descriptor["file"] + require_keys(item, ["path", "sha256"]) + @test item["path"] isa String + @test item["sha256"] isa String + @test safe_relative(item["path"]) + @test occursin(SHA256_PATTERN, item["sha256"]) + @test item["path"] ∉ producer_paths + @test control_snapshot(snapshots, + item["path"]).sha256 == item["sha256"] + push!(producer_paths, item["path"]) + end + @test producer_paths == expected_producer_files + authority = only(filter(item -> item["id"] == "n6-independent-model", + capabilities["authority"])) + @test authority["revision"] == manifest[ + "model_producer_descriptor_sha256"] + for evidence in manifest["frozen_evidence"] + @test control_snapshot(snapshots, + evidence["schema_file"]).sha256 == evidence["schema_sha256"] + @test normpath(evidence["fixture_manifest_file"]) == + normpath(manifest["fixture_manifest_file"]) + if evidence["storage"] == "checked-in" + @test file_sha256(checked_file(REPO_DIR, evidence["file"])) == + evidence["sha256"] + else + @test !ispath(joinpath(REPO_DIR, evidence["file"])) + end + end + validate_python_descriptors(manifest, capabilities, snapshots) + validate_arrow_descriptor(manifest, capabilities, snapshots) + validate_parquet_java_descriptor(manifest, capabilities, snapshots) + validate_parquet_jl_descriptor(manifest, capabilities, snapshots) + normalized_schema = String(copy(control_snapshot(snapshots, + manifest["evidence_schema_file"]).payload)) + @test !occursin("\"manifest_sha256\"", normalized_schema) + @test !occursin("\"artifact_manifest_sha256\"", normalized_schema) + for field in ("plan_sha256", "capabilities_sha256", + "fixture_manifest_sha256", "corpus_manifest_sha256", + "evidence_schema_sha256", "toolchain_sha256") + @test occursin("\"$field\"", normalized_schema) + end + entries = artifact_entries(control_snapshot(snapshots, + manifest["artifact_manifest_file"]).payload) + mutable_files = Set{String}() + all_evidence = vcat(manifest["frozen_evidence"], manifest["planned_evidence"]) + @test length(unique(item["file"] for item in all_evidence)) == + length(all_evidence) + for evidence in all_evidence + safe_evidence_file(evidence["file"]) || + error("unsafe evidence exclusion: " * evidence["file"]) + relative = relpath(joinpath(REPO_DIR, evidence["file"]), N6_DIR) + push!(mutable_files, + replace(relative, Base.Filesystem.path_separator => '/')) + end + generated_files = Set{String}() + for item in fixtures["generated_case"] + safe_generated_output(item["output_file"]) || + error("unsafe generated output exclusion: " * item["output_file"]) + push!(generated_files, item["output_file"]) + end + @test length(generated_files) == length(fixtures["generated_case"]) + union!(mutable_files, generated_files) + @test isempty(intersect(Set(first.(entries)), mutable_files)) + @test "artifacts.sha256" ∉ first.(entries) + @test "manifest.toml" ∉ first.(entries) + @test first.(entries) == intended_artifacts(; mutable_files) + for (relative, digest) in entries + @test control_snapshot(snapshots, + "test/conformance/n6/" * relative).sha256 == digest + end + for relative in ( + "test/conformance/n6/oracles/raw-java/check.sh", + "test/conformance/n6/oracles/raw-java/run.sh", + "test/conformance/n6/oracles/raw-java/scripts/build.sh", + "test/conformance/n6/oracles/raw-java/scripts/generate.sh") + script = String(copy(control_snapshot(snapshots, relative).payload)) + for line in eachline(IOBuffer(script)) + occursin(r"\"\$root/[^\"]+\.sh\"", line) || continue + invocation = strip(line) + startswith(invocation, "#") && continue + @test startswith(invocation, "/bin/bash ") || + startswith(invocation, "source ") + end + end + return +end + +function validate_source_roots(manifest, fixtures) + for source in manifest["source"] + environment_name = source["root_env"] + haskey(ENV, environment_name) || error("gate requires $environment_name") + root = realpath(ENV[environment_name]) + require_gate(strip(read(isolated_command( + `/usr/bin/git -C $root rev-parse HEAD`), String)) == + source["revision"], "source revision differs: " * source["id"]) + require_gate(isempty(strip(read(isolated_command( + `/usr/bin/git -C $root status --porcelain`), String))), + "source worktree is dirty: " * source["id"]) + require_gate(strip(read(isolated_command( + `/usr/bin/git -C $root remote get-url origin`), String)) == + source["url"], "source origin differs: " * source["id"]) + if !isempty(source["tag"]) + tag_reference = "refs/tags/" * source["tag"] + peeled_reference = tag_reference * "^{}" + require_gate(strip(read(isolated_command( + `/usr/bin/git -C $root rev-parse $tag_reference`), String)) == + source["tag_revision"], "source tag differs: " * source["id"]) + require_gate(strip(read(isolated_command( + `/usr/bin/git -C $root rev-parse $peeled_reference`), + String)) == + source["revision"], + "source peeled tag differs: " * source["id"]) + end + for pinned in source["files"] + require_gate(file_sha256(checked_file(root, pinned["file"])) == + pinned["sha256"], "source file hash differs: " * + source["id"] * ":" * pinned["file"]) + end + end + testing_source = only(filter(item -> item["id"] == "parquet-testing", + manifest["source"])) + testing_root = ENV[testing_source["root_env"]] + for fixture in fixtures["fixture"] + file = checked_file(testing_root, fixture["file"]) + require_gate(filesize(file) == fixture["size"], + "fixture size differs: " * fixture["id"]) + require_gate(file_sha256(file) == fixture["sha256"], + "fixture hash differs: " * fixture["id"]) + end + return +end + +function parse_shell_assignments(file::AbstractString) + assignments = Dict{String,String}() + for (line_number, line) in enumerate(eachline(file)) + match_result = match(r"^([A-Z][A-Z0-9_]*)=(\S+)$", line) + match_result === nothing && + error("invalid shell assignment at line $line_number") + name, value = match_result.captures + haskey(assignments, name) && + error("duplicate shell assignment: $name") + assignments[name] = value + end + return assignments +end + +function raw_java_download_snapshots(manifest, gate_root) + cache_input = get(ENV, "PARQUET_N6_RAW_JAVA_DOWNLOAD_CACHE", "") + isempty(cache_input) && + error("gate requires PARQUET_N6_RAW_JAVA_DOWNLOAD_CACHE") + islink(cache_input) && + error("raw Java download cache is a symbolic link") + isdir(cache_input) || error("raw Java download cache is absent") + downloads = realpath(cache_input) + pins = parse_shell_assignments(gate_file(gate_root, + "test/conformance/n6/oracles/raw-java/toolchain.env")) + expected = [ + ("OpenJDK21U-jdk_aarch64_mac_hotspot_21.0.8_9.tar.gz", + "RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_SHA256"), + ("homebrew-thrift-bd296b14f19462baf03d5d96920209087ca99fa0.rb", + "RAW_JAVA_HOMEBREW_FORMULA_SHA256"), + ("libthrift-0.23.0.jar", "RAW_JAVA_LIBTHRIFT_SHA256"), + ("libthrift-0.23.0.pom", "RAW_JAVA_LIBTHRIFT_POM_SHA256"), + ("slf4j-api-1.7.36.jar", "RAW_JAVA_SLF4J_API_SHA256"), + ("slf4j-nop-1.7.36.jar", "RAW_JAVA_SLF4J_NOP_SHA256"), + ("thrift-0.23.0-dd6ed015e1b7a980c3dfa2b0dd1c01d563a8cf73bdb7f3de87d0cc1656fc1e1b.bottle.tar.gz", + "RAW_JAVA_BOTTLE_DARWIN_ARM64_SEQUOIA_SHA256"), + ("thrift-0.23.0.tar.gz", "RAW_JAVA_THRIFT_SOURCE_SHA256"), + ] + files = checked_directory_files(downloads, first.(expected)) + snapshots = Dict{String,FileSnapshot}() + total_bytes = 0 + for ((name, pin), file) in zip(expected, files) + haskey(pins, pin) || error("raw Java pin is absent: $pin") + occursin(SHA256_PATTERN, pins[pin]) || + error("raw Java pin is invalid: $pin") + payload = read_bounded_regular_bytes(file, RAW_JAVA_DOWNLOAD_LIMIT) + length(payload) <= RAW_JAVA_DOWNLOAD_TOTAL_LIMIT - total_bytes || + error("raw Java downloads exceed their total byte limit") + total_bytes += length(payload) + digest = bytes2hex(SHA.sha256(payload)) + require_gate(digest == pins[pin], + "raw Java cached download hash differs: $name") + snapshots[name] = FileSnapshot(file, payload, digest) + end + require_gate(pins["RAW_JAVA_JDK_DARWIN_ARM64_SEQUOIA_SHA256"] == + toolchain_artifact(manifest, "raw-java", "temurin-jdk-archive"), + "raw Java JDK archive pin differs from the manifest") + for (pin, artifact) in ( + ("RAW_JAVA_LIBTHRIFT_SHA256", "libthrift-0.23.0.jar"), + ("RAW_JAVA_LIBTHRIFT_POM_SHA256", "libthrift-0.23.0.pom"), + ("RAW_JAVA_SLF4J_API_SHA256", "slf4j-api-1.7.36.jar"), + ("RAW_JAVA_SLF4J_NOP_SHA256", "slf4j-nop-1.7.36.jar")) + require_gate(pins[pin] == toolchain_artifact(manifest, "raw-java", + artifact), "raw Java cached download pin differs: $artifact") + end + return snapshots +end + +function with_raw_java_build(f, manifest, gate_root) + snapshots = raw_java_download_snapshots(manifest, gate_root) + return mktempdir() do directory + build = joinpath(directory, "build") + downloads = joinpath(build, "downloads") + mkpath(downloads) + private_files = Dict{String,String}() + for (name, snapshot) in snapshots + file = joinpath(downloads, name) + open(file, "w") do io + write(io, snapshot.payload) + return + end + chmod(file, 0o400) + private_files[name] = file + end + chmod(downloads, 0o500) + try + result = f(build) + for (name, snapshot) in snapshots + require_gate(file_sha256(private_files[name]) == snapshot.sha256, + "private raw Java download changed: $name") + require_gate(file_sha256(snapshot.path) == snapshot.sha256, + "canonical raw Java download changed: $name") + end + return result + finally + chmod(downloads, 0o700) + for file in values(private_files) + chmod(file, 0o600) + end + end + end +end + +function validate_raw_java_snapshot(manifest, fixtures, python, isolated, + gate_root, raw_build, julia_runtime) + evidence = only(filter(item -> item["id"] == "raw-java-apache-corpus", + manifest["frozen_evidence"])) + evidence_schema = gate_file(gate_root, evidence["schema_file"]) + fixture_manifest = gate_file(gate_root, evidence["fixture_manifest_file"]) + raw_dir = joinpath(gate_root, "test", "conformance", "n6", "oracles", + "raw-java") + raw_environment = "PARQUET_N6_RAW_JAVA_BUILD_DIR" => raw_build + run(isolated(`/bin/bash $(joinpath(raw_dir, "check.sh"))`, + raw_environment)) + testing_source = only(filter(item -> item["id"] == "parquet-testing", + manifest["source"])) + testing_root = ENV[testing_source["root_env"]] + selected = sort(fixtures["fixture"]; by=item -> item["file"]) + inputs = [checked_file(testing_root, item["file"]) for item in selected] + labels = basename.(inputs) + require_gate(length(labels) == length(unique(labels)), + "raw corpus labels are ambiguous") + validation = raw""" +import json +import pathlib +import sys +import tomllib +import jsonschema + +schema_path, fixtures_path, evidence_path, expected_count, max_file_bytes, max_line_bytes = sys.argv[1:] +schema = json.loads(pathlib.Path(schema_path).read_text()) +validator_class = jsonschema.validators.validator_for(schema) +validator_class.check_schema(schema) +validator = validator_class(schema) +fixtures = tomllib.loads(pathlib.Path(fixtures_path).read_text()) +expected = { + pathlib.Path(item["file"]).name: (item["sha256"], item["size"]) + for item in fixtures["fixture"] +} +seen = set() +evidence = pathlib.Path(evidence_path) +if evidence.stat().st_size > int(max_file_bytes): + raise ValueError("raw evidence exceeds its byte limit") +with open(evidence, "rb") as stream: + line_number = 0 + while True: + line = stream.readline(int(max_line_bytes) + 1) + if not line: + break + line_number += 1 + if len(line) > int(max_line_bytes): + raise ValueError(f"line {line_number} exceeds its byte limit") + if not line.endswith(b"\n"): + raise ValueError(f"line {line_number} lacks its final newline") + record = json.loads(line.decode("utf-8")) + validator.validate(record) + label = record["file"] + if label not in expected or label in seen: + raise ValueError(f"unexpected or duplicate raw label: {label}") + if (record["file_sha256"], record["file_size"]) != expected[label]: + raise ValueError(f"raw file identity mismatch: {label}") + seen.add(label) +if len(seen) != int(expected_count) or seen != set(expected): + raise ValueError("raw corpus evidence is incomplete") +""" + mktempdir() do output_dir + outputs = [joinpath(output_dir, "raw-$index.jsonl") for index in 1:2] + for output in outputs + arguments = String["/bin/bash", joinpath(raw_dir, "run.sh"), "scan"] + for input in inputs + append!(arguments, ["--input", input]) + end + append!(arguments, ["--output", output]) + run(isolated(Cmd(arguments), raw_environment)) + output_size = filesize(output) + 0 < output_size <= manifest["evidence_limits"]["max_file_bytes"] || + error("raw evidence has an invalid byte size") + require_gate(file_sha256(output) == evidence["sha256"], + "raw evidence hash differs") + command = Cmd(String[python, "-I", "-c", validation, evidence_schema, + fixture_manifest, output, string(evidence["record_count"]), + string(manifest["evidence_limits"]["max_file_bytes"]), + string(manifest["evidence_limits"]["max_line_bytes"])]) + run(isolated(command)) + end + require_gate(read(outputs[1]) == read(outputs[2]), + "raw scanner output is not deterministic") + normalized = joinpath(gate_root, "test", "conformance", "n6", "evidence", + "raw-java-apache-corpus.normalized.jsonl") + model = joinpath(gate_root, "test", "conformance", "n6", "evidence", + "independent-model.normalized.jsonl") + normalizers = joinpath(gate_root, "test", "conformance", "n6", + "normalizers") + run(isolated(Cmd(String[ + python, "-I", "-B", joinpath(normalizers, "normalize_raw.py"), + "--input", outputs[1], "--output", normalized, "--check", + ]))) + julia_executable = checked_file(julia_runtime, "bin/julia") + run(isolated(Cmd(String[ + python, "-I", "-B", joinpath(normalizers, "normalize_model.py"), + "--input", normalized, "--raw-input", outputs[1], + "--output", model, "--julia-executable", julia_executable, "--check", + ]))) + return + end + generated = sort(fixtures["generated_case"]; + by=item -> item["output_file"]) + generated_inputs = [gate_file(gate_root, + "test/conformance/n6/" * item["output_file"]) for item in generated] + mktempdir() do output_dir + outputs = [joinpath(output_dir, "generated-$index.jsonl") + for index in 1:2] + for output in outputs + arguments = String["/bin/bash", joinpath(raw_dir, "run.sh"), + "scan"] + for input in generated_inputs + append!(arguments, ["--input", input]) + end + append!(arguments, ["--output", output]) + run(isolated(Cmd(arguments), raw_environment)) + output_size = filesize(output) + 0 < output_size <= manifest["evidence_limits"]["max_file_bytes"] || + error("generated raw evidence has an invalid byte size") + end + require_gate(read(outputs[1]) == read(outputs[2]), + "generated raw scanner output is not deterministic") + generated_evidence = evidence_entry(manifest, + "normalized-raw-java-generated") + normalized = gate_file(gate_root, generated_evidence["file"]) + normalizer = gate_file(gate_root, + "test/conformance/n6/normalizers/normalize_raw.py") + run(isolated(Cmd(String[ + python, "-I", "-B", normalizer, + "--fixture-set", "generated", "--input", outputs[1], + "--output", normalized, "--check", + ]))) + return + end + return +end + +function validate_raw_java_gate(manifest, fixtures, python, isolated, gate_root, + julia_runtime) + with_raw_java_build(manifest, gate_root) do raw_build + validate_raw_java_snapshot(manifest, fixtures, python, isolated, + gate_root, raw_build, julia_runtime) + return + end + return +end + +function validator_runtime_inputs(manifest) + archive_input = get(ENV, "PARQUET_N6_VALIDATOR_PYTHON_ARCHIVE", "") + isempty(archive_input) && + error("gate requires PARQUET_N6_VALIDATOR_PYTHON_ARCHIVE") + islink(archive_input) && + error("validator Python archive is a symbolic link") + isfile(archive_input) || error("validator Python archive is absent") + archive = realpath(archive_input) + archive_payload = read_bounded_regular_bytes(archive, + VALIDATOR_DISTRIBUTION_ARCHIVE_LIMIT) + require_gate(bytes2hex(SHA.sha256(archive_payload)) == + toolchain_artifact(manifest, "jsonschema-validator", + "cpython-distribution-archive"), + "validator Python archive hash differs") + validator_wheels = get(ENV, "PARQUET_N6_VALIDATOR_WHEEL_DIR", "") + isempty(validator_wheels) && error("gate requires PARQUET_N6_VALIDATOR_WHEEL_DIR") + islink(validator_wheels) && error("validator wheel root is a symbolic link") + isdir(validator_wheels) || error("validator wheel root is absent") + wheel_root = realpath(validator_wheels) + validator_names = [ + "jsonschema-4.26.0-py3-none-any.whl", + "attrs-25.4.0-py3-none-any.whl", + "jsonschema_specifications-2025.9.1-py3-none-any.whl", + "referencing-0.37.0-py3-none-any.whl", + "rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", + ] + validator_files = checked_directory_files(wheel_root, validator_names) + wheel_payloads = Dict{String,Vector{UInt8}}() + for (artifact_name, file) in zip(validator_names, validator_files) + payload = read_bounded_regular_bytes(file, 64 * 1024 * 1024) + require_gate(bytes2hex(SHA.sha256(payload)) == + toolchain_artifact(manifest, "jsonschema-validator", artifact_name), + "validator wheel hash differs: $artifact_name") + wheel_payloads[artifact_name] = payload + end + return archive_payload, validator_names, wheel_payloads +end + +function checked_runtime_directory(root::AbstractString, parts::String...) + runtime_root = realpath(root) + current = runtime_root + for part in parts + candidate = joinpath(current, part) + (ispath(candidate) || islink(candidate)) || + error("Python distribution lacks " * join(parts, '/')) + islink(candidate) && error("Python runtime path is a symbolic link: " * + relpath(candidate, runtime_root)) + isdir(candidate) || error("Python runtime path is not a directory: " * + relpath(candidate, runtime_root)) + resolved = realpath(candidate) + startswith(resolved, runtime_root * Base.Filesystem.path_separator) || + error("Python runtime path escapes its root") + current = resolved + end + return current +end + +function prune_python_runtime!(root::AbstractString, + version_directory::AbstractString) + runtime_root = realpath(root) + site_packages = checked_runtime_directory(runtime_root, "lib", + version_directory, + "site-packages") + rm(site_packages; recursive=true) + for (directory, directories, files) in walkdir(runtime_root; + topdown=false, follow_symlinks=false) + for name in files + endswith(name, ".pyc") || continue + file = joinpath(directory, name) + islink(file) && error("Python bytecode cache is a symbolic link") + rm(file) + end + for name in directories + name == "__pycache__" || continue + cache = joinpath(directory, name) + islink(cache) && error("Python cache directory is a symbolic link") + rm(cache; recursive=true) + end + end + return +end + +function prune_interop_python!(root::AbstractString) + prune_python_runtime!(root, "python3.12") + return +end + +function lock_python_runtime!(root::AbstractString, + executable_relative::AbstractString) + runtime_root = realpath(root) + for (directory, directories, files) in walkdir(runtime_root; + follow_symlinks=false) + for name in files + file = joinpath(directory, name) + islink(file) && continue + chmod(file, 0o400) + end + for name in directories + child = joinpath(directory, name) + islink(child) && continue + chmod(child, 0o500) + end + end + chmod(checked_file(runtime_root, executable_relative), 0o500) + chmod(runtime_root, 0o500) + return +end + +function lock_interop_python!(root::AbstractString) + lock_python_runtime!(root, "bin/python3.12") + return +end + +function unlock_interop_python!(root::AbstractString) + runtime_root = realpath(root) + chmod(runtime_root, 0o700) + for (directory, directories, files) in walkdir(runtime_root; + follow_symlinks=false) + for name in directories + child = joinpath(directory, name) + islink(child) && continue + chmod(child, 0o700) + end + for name in files + file = joinpath(directory, name) + islink(file) && continue + chmod(file, 0o600) + end + end + return +end + +function with_validator_runtime(f, manifest) + archive_payload, wheel_names, wheel_payloads = + validator_runtime_inputs(manifest) + return mktempdir() do directory + archive = joinpath(directory, "cpython.tar.gz") + open(archive, "w") do io + write(io, archive_payload) + return + end + chmod(archive, 0o400) + extract_root = joinpath(directory, "extract") + mkdir(extract_root) + run(isolated_command(`/usr/bin/tar -xzf $archive -C $extract_root`)) + require_gate(readdir(extract_root) == ["python"], + "validator Python archive top-level contents differ") + runtime_root = joinpath(extract_root, "python") + islink(runtime_root) && + error("validator Python runtime root is a symbolic link") + isdir(runtime_root) || + error("validator Python runtime root is absent") + prune_python_runtime!(runtime_root, "python3.14") + runtime_inventory = validate_bounded_tree(runtime_root; + max_entries=VALIDATOR_TREE_ENTRY_LIMIT, + max_file_bytes=VALIDATOR_TREE_FILE_LIMIT, + max_total_bytes=VALIDATOR_TREE_TOTAL_LIMIT) + require_gate(runtime_inventory.sha256 == toolchain_artifact(manifest, + "jsonschema-validator", + "cpython-clean-tree-sha256-v1"), + "validator Python snapshot tree hash differs") + base_python = checked_file(runtime_root, "bin/python3.14") + require_gate(file_sha256(base_python) == toolchain_artifact(manifest, + "jsonschema-validator", "cpython-executable"), + "validator Python snapshot executable differs") + wheel_root = joinpath(directory, "wheels") + mkdir(wheel_root) + validator_files = String[] + for name in wheel_names + file = joinpath(wheel_root, name) + open(file, "w") do io + write(io, wheel_payloads[name]) + return + end + chmod(file, 0o400) + push!(validator_files, file) + end + chmod(wheel_root, 0o500) + try + lock_python_runtime!(runtime_root, "bin/python3.14") + result = f(base_python, validator_files) + require_gate(validate_bounded_tree(runtime_root; + max_entries=VALIDATOR_TREE_ENTRY_LIMIT, + max_file_bytes=VALIDATOR_TREE_FILE_LIMIT, + max_total_bytes=VALIDATOR_TREE_TOTAL_LIMIT).sha256 == + toolchain_artifact(manifest, + "jsonschema-validator", + "cpython-clean-tree-sha256-v1"), + "validator Python snapshot changed after use") + for (name, file) in zip(wheel_names, validator_files) + require_gate(file_sha256(file) == toolchain_artifact(manifest, + "jsonschema-validator", name), + "validator wheel snapshot changed after use: $name") + end + return result + finally + unlock_interop_python!(runtime_root) + chmod(wheel_root, 0o700) + for file in validator_files + chmod(file, 0o600) + end + end + end +end + +function with_interop_runtime(f, manifest) + archive_input = get(ENV, "PARQUET_N6_INTEROP_PYTHON_ARCHIVE", "") + isempty(archive_input) && + error("gate requires PARQUET_N6_INTEROP_PYTHON_ARCHIVE") + isfile(archive_input) || error("Python distribution archive is absent") + islink(archive_input) && + error("Python distribution archive is a symbolic link") + archive = realpath(archive_input) + archive_payload = read_bounded_regular_bytes(archive, + CPYTHON_DISTRIBUTION_ARCHIVE_LIMIT) + require_gate(bytes2hex(SHA.sha256(archive_payload)) == + toolchain_artifact(manifest, "python-interop", + "cpython-distribution-archive"), + "Python distribution archive hash differs") + interop_wheels = get(ENV, "PARQUET_N6_INTEROP_WHEEL_DIR", "") + isempty(interop_wheels) && error("gate requires PARQUET_N6_INTEROP_WHEEL_DIR") + interop_names = [ + "pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", + "duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", + ] + interop_files = checked_directory_files(interop_wheels, interop_names) + for (artifact_name, file) in zip(interop_names, interop_files) + require_gate(file_sha256(file) == + toolchain_artifact(manifest, "python-interop", artifact_name), + "Python interop wheel hash differs: $artifact_name") + end + wheels = Dict(name => file + for (name, file) in zip(interop_names, interop_files)) + return mktempdir() do directory + archive_snapshot = joinpath(directory, "cpython.tar.gz") + open(archive_snapshot, "w") do io + write(io, archive_payload) + return + end + chmod(archive_snapshot, 0o400) + extract_root = joinpath(directory, "extract") + mkdir(extract_root) + run(isolated_command( + `/usr/bin/tar -xzf $archive_snapshot -C $extract_root`)) + require_gate(readdir(extract_root) == ["python"], + "Python archive top-level contents differ") + interop_root = joinpath(extract_root, "python") + islink(interop_root) && error("Python runtime root is a symbolic link") + isdir(interop_root) || error("Python runtime root is not a directory") + prune_interop_python!(interop_root) + require_gate(tree_sha256(interop_root) == toolchain_artifact(manifest, + "python-interop", "cpython-clean-tree-sha256-v1"), + "clean Python runtime tree hash differs") + interop_python = checked_file(interop_root, "bin/python3.12") + require_gate(file_sha256(interop_python) == toolchain_artifact(manifest, + "python-interop", "cpython-executable"), + "interop Python executable hash differs") + require_gate(!ispath(joinpath(interop_root, "lib", "python3.12", + "site-packages")), "Python site-packages survived pruning") + try + lock_interop_python!(interop_root) + return f(interop_python, wheels) + finally + unlock_interop_python!(interop_root) + end + end +end + +function isolated_command(command, extra::Pair...) + environment = Dict{String,String}( + "GIT_CONFIG_GLOBAL" => "/dev/null", + "GIT_CONFIG_NOSYSTEM" => "1", + "HOME" => "/var/empty", + "LANG" => "C", + "LC_ALL" => "C", + "NO_COLOR" => "1", + "PATH" => "/usr/bin:/bin:/usr/sbin:/sbin", + "TERM" => "dumb", + "TMPDIR" => tempdir(), + "TZ" => "UTC", + ) + for (name, value) in extra + environment[String(name)] = String(value) + end + return setenv(command, environment) +end + +function isolated_python_command(command, extra::Pair...) + return isolated_command(command, + "PIP_CONFIG_FILE" => "/dev/null", + "PIP_DISABLE_PIP_VERSION_CHECK" => "1", + "PIP_NO_INDEX" => "1", + "PYTHONHASHSEED" => "0", + "PYTHONNOUSERSITE" => "1", + "PYTHONPATH" => "", + extra...) +end + +function bootstrap_validator_environment(environment, base_python, validator_files) + flags = strip(read(isolated_python_command( + `$base_python -I -B -S -c "import sys; print(f'{sys.flags.isolated}|{sys.flags.no_site}|{sys.flags.dont_write_bytecode}')"`), + String)) + require_gate(flags == "1|1|1", "validator Python isolation flags differ") + run(isolated_python_command(`$base_python -I -B -S -m venv $environment`)) + python = joinpath(environment, "bin", "python") + install = Cmd(vcat([python, "-I", "-m", "pip", "install", "--no-cache-dir", + "--no-deps", "--no-index"], validator_files)) + run(isolated_python_command(install)) + versions = strip(read(isolated_python_command( + `$python -I -c "import importlib.metadata as m; print('|'.join(m.version(x) for x in ('jsonschema','attrs','jsonschema-specifications','referencing','rpds-py')))"`), + String)) + require_gate(versions == "4.26.0|25.4.0|2025.9.1|0.37.0|0.30.0", + "validator package versions differ") + return python +end + +function validator_arguments(python, gate_root) + return String[ + python, + "-I", + gate_file(gate_root, "test/conformance/n6/validate_evidence.py"), + "--schema", gate_file(gate_root, + "test/conformance/n6/evidence.schema.json"), + "--manifest", gate_file(gate_root, + "test/conformance/n6/manifest.toml"), + "--capabilities", gate_file(gate_root, + "test/conformance/n6/capabilities.toml"), + "--fixtures", gate_file(gate_root, + "test/conformance/n6/fixtures.toml"), + ] +end + +function normalized_gate_command(arguments::Vector{String}, + normalized::Vector{String}, gate_root::AbstractString) + all(safe_evidence_file, normalized) || + error("gate evidence arguments must be canonical repository-relative paths") + return Cmd(Cmd(vcat(arguments, String["--gate"], normalized)); dir=gate_root) +end + +function validate_normalized_gate(manifest, python, gate_root) + arguments = validator_arguments(python, gate_root) + run(isolated_python_command(Cmd(vcat(arguments, ["--self-test"])))) + normalized = sort!(String[item["file"] + for item in manifest["frozen_evidence"] + if item["format"] == "normalized-jsonl"]) + gate = normalized_gate_command(arguments, normalized, gate_root) + if isempty(normalized) + mktemp() do _, output + closed = pipeline(ignorestatus(isolated_python_command(gate)); + stdout=output, stderr=output) + process = run(closed) + flush(output) + seekstart(output) + message = read(output, String) + require_gate(!success(process), + "empty normalized evidence unexpectedly passed") + require_gate(startswith(message, + "N6 evidence validation failed: " * + "missing passing capability evidence:"), + "empty normalized evidence failed for the wrong reason") + return + end + else + run(isolated_python_command(gate)) + end + return +end + +function validate_required_raw_gate(manifest, fixtures, python, gate_root, + julia_runtime) + validate_raw_java_gate(manifest, fixtures, python, isolated_python_command, + gate_root, julia_runtime) + return +end + +function interop_harness_arguments(manifest, gate_root) + testing_source = only(filter(item -> item["id"] == "parquet-testing", + manifest["source"])) + testing_root = ENV[testing_source["root_env"]] + arguments = String[ + "--repository", gate_root, + "--manifest", gate_file(gate_root, + "test/conformance/n6/manifest.toml"), + "--capabilities", gate_file(gate_root, + "test/conformance/n6/capabilities.toml"), + "--fixtures", gate_file(gate_root, + "test/conformance/n6/fixtures.toml"), + "--raw-evidence", gate_file(gate_root, + "test/conformance/n6/evidence/" * + "raw-java-apache-corpus.normalized.jsonl"), + "--corpus-root", testing_root, + ] + return arguments +end + +function evidence_entry(manifest, id::AbstractString) + matches = filter(item -> item["id"] == id, + vcat(manifest["frozen_evidence"], manifest["planned_evidence"])) + return only(matches) +end + +function draft_argument(evidence) + return evidence["status"] == "planned" ? ["--draft"] : String[] +end + +function validate_python_interop_gate(manifest, python, wheels, gate_root) + shared = interop_harness_arguments(manifest, gate_root) + configurations = [ + ("pyarrow", "pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl"), + ("duckdb", "duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl"), + ] + for (producer, wheel_name) in configurations + evidence = evidence_entry(manifest, "normalized-$producer") + arguments = String[ + python, "-I", "-B", "-S", + gate_file(gate_root, + "test/conformance/n6/harnesses/$producer.py"), + shared..., + "--descriptor", gate_file(gate_root, + "test/conformance/n6/toolchains/$producer.toml"), + "--wheel", wheels[wheel_name], + "--output", gate_file(gate_root, evidence["file"]), + "--check", + ] + append!(arguments, draft_argument(evidence)) + run(isolated_python_command(Cmd(arguments))) + end + return +end + +function arrow_docker_executable() + input = get(ENV, "PARQUET_N6_DOCKER", "") + isempty(input) && error("gate requires PARQUET_N6_DOCKER") + isabspath(input) || error("Arrow Rust Docker executable is not absolute") + islink(input) && error("Arrow Rust Docker executable is a symbolic link") + isfile(input) || error("Arrow Rust Docker executable is absent") + executable = realpath(input) + stat(executable).mode & 0o111 != 0 || + error("Arrow Rust Docker executable is not executable") + return executable +end + +function validate_arrow_interop_gate(manifest, python, gate_root) + evidence = evidence_entry(manifest, "normalized-arrow-rs") + arguments = String[ + python, "-I", "-B", "-S", + gate_file(gate_root, + "test/conformance/n6/oracles/arrow-rs/run.py"), + interop_harness_arguments(manifest, gate_root)..., + "--descriptor", gate_file(gate_root, + "test/conformance/n6/oracles/arrow-rs/toolchain.toml"), + "--output", gate_file(gate_root, evidence["file"]), + "--docker", arrow_docker_executable(), + "--check", + ] + append!(arguments, draft_argument(evidence)) + run(isolated_python_command(Cmd(arguments))) + return +end + +function parquet_java_jdk_input() + jdk_input = get(ENV, "PARQUET_N6_JAVA_JDK_ROOT", "") + isempty(jdk_input) && error("gate requires PARQUET_N6_JAVA_JDK_ROOT") + islink(jdk_input) && error("Parquet Java JDK root is a symbolic link") + isdir(jdk_input) || error("Parquet Java JDK root is absent") + root = realpath(jdk_input) + inventory = validate_bounded_tree(root; + max_entries=JDK_TREE_ENTRY_LIMIT, + max_file_bytes=JDK_TREE_FILE_LIMIT, + max_total_bytes=JDK_TREE_TOTAL_LIMIT) + return root, inventory +end + +function validate_parquet_java_jdk_snapshot(manifest, jdk_root) + inventory = validate_bounded_tree(jdk_root; + max_entries=JDK_TREE_ENTRY_LIMIT, + max_file_bytes=JDK_TREE_FILE_LIMIT, + max_total_bytes=JDK_TREE_TOTAL_LIMIT) + require_gate(inventory.sha256 == toolchain_artifact(manifest, + "parquet-java-interop", "temurin-tree-sha256-v1"), + "Parquet Java JDK tree hash differs") + for (relative, artifact_name) in ( + ("bin/java", "temurin-java"), + ("bin/javac", "temurin-javac"), + ("release", "temurin-release")) + require_gate(file_sha256(checked_file(jdk_root, relative)) == + toolchain_artifact(manifest, "parquet-java-interop", artifact_name), + "Parquet Java JDK artifact differs: $artifact_name") + end + return +end + +function with_parquet_java_jdk(f, manifest) + source_root, source_inventory = parquet_java_jdk_input() + return mktempdir() do directory + jdk_root = joinpath(directory, "jdk") + cp(source_root, jdk_root; follow_symlinks=false) + jdk_root = realpath(jdk_root) + require_gate(validate_bounded_tree(source_root; + max_entries=JDK_TREE_ENTRY_LIMIT, + max_file_bytes=JDK_TREE_FILE_LIMIT, + max_total_bytes=JDK_TREE_TOTAL_LIMIT) == source_inventory, + "Parquet Java JDK source tree changed while copied") + validate_parquet_java_jdk_snapshot(manifest, jdk_root) + try + lock_python_runtime!(jdk_root, "bin/java") + chmod(checked_file(jdk_root, "bin/javac"), 0o500) + result = f(jdk_root) + validate_parquet_java_jdk_snapshot(manifest, jdk_root) + return result + finally + unlock_interop_python!(jdk_root) + end + end +end + +function validate_parquet_java_interop_gate(manifest, python, jdk_root, + gate_root) + evidence = evidence_entry(manifest, "normalized-parquet-java") + java_source = only(filter(item -> item["id"] == "parquet-java", + manifest["source"])) + java_root = ENV[java_source["root_env"]] + arguments = String[ + python, "-I", "-B", "-S", + gate_file(gate_root, + "test/conformance/n6/oracles/parquet-java/run.py"), + interop_harness_arguments(manifest, gate_root)..., + "--descriptor", gate_file(gate_root, + "test/conformance/n6/oracles/parquet-java/toolchain.toml"), + "--java-root", java_root, + "--jdk-root", jdk_root, + "--output", gate_file(gate_root, evidence["file"]), + "--check", + ] + append!(arguments, draft_argument(evidence)) + run(isolated_python_command(Cmd(arguments))) + return +end + +function validate_python_gate(manifest, capabilities, fixtures, snapshots) + with_validator_runtime(manifest) do base_python, validator_files + with_gate_repository(snapshots) do gate_root + with_verified_parquet_jl_runtime(manifest, capabilities, snapshots, + gate_root) do julia_runtime + julia_executable = checked_file(julia_runtime, "bin/julia") + mktempdir() do environment + python = bootstrap_validator_environment(environment, + base_python, validator_files) + run(isolated_python_command(Cmd(String[ + python, "-I", "-B", + gate_file(gate_root, + "test/conformance/n6/normalizers/runtests.py"), + ]), "PARQUET_N6_TEST_JULIA_EXECUTABLE" => julia_executable)) + validate_required_raw_gate(manifest, fixtures, python, + gate_root, julia_runtime) + with_interop_runtime(manifest) do interop_python, + interop_wheels + run(isolated_python_command(Cmd(String[ + interop_python, "-I", "-B", "-S", + gate_file(gate_root, + "test/conformance/n6/harnesses/test_common.py"), + ]))) + validate_python_interop_gate(manifest, interop_python, + interop_wheels, gate_root) + writable = String["test/conformance/n6/evidence"] + with_gate_repository(snapshots; + writable_directories=writable) do oracle_root + validate_arrow_interop_gate(manifest, + interop_python, oracle_root) + with_parquet_java_jdk(manifest) do jdk_root + validate_parquet_java_interop_gate(manifest, + interop_python, jdk_root, oracle_root) + return + end + return + end + return + end + validate_normalized_gate(manifest, python, gate_root) + return + end + return + end + return + end + return + end + return +end + +function validate_running_julia(manifest) + expected_id = VERSION < v"1.11" ? "julia-1.10" : "julia-1.12" + expected_name = VERSION < v"1.11" ? "julia-1.10.11-executable" : + "julia-1.12.6-executable" + require_gate(file_sha256(joinpath(Sys.BINDIR, "julia")) == + toolchain_artifact(manifest, expected_id, expected_name), + "running Julia executable hash differs") + runtime_root = normpath(joinpath(Sys.BINDIR, "..")) + require_gate(tree_sha256(runtime_root) == toolchain_artifact(manifest, + expected_id, replace(expected_name, "-executable" => + "-runtime-tree-sha256-v1")), + "running Julia runtime tree hash differs") + return +end + +function test_path_safety_helpers() + oracles = ("arrow-rs", "raw-java", "parquet-java") + mktempdir() do source_root + relative = "test/conformance/n6/evidence/example.jsonl" + source = joinpath(source_root, relative) + mkpath(dirname(source)) + payload = Vector{UInt8}("evidence\n") + write(source, payload) + snapshots = Dict(relative => FileSnapshot(source, payload, + bytes2hex(SHA.sha256(payload)))) + writable = String[dirname(relative)] + with_gate_repository(snapshots; + writable_directories=writable) do gate_root + output = gate_file(gate_root, relative) + @test stat(dirname(output)).mode & 0o200 != 0 + @test stat(output).mode & 0o222 == 0 + mktempdir(dirname(output)) do workspace + @test isdir(workspace) + return + end + @test read(output) == payload + return + end + @test read(source) == payload + @test_throws ErrorException with_gate_repository(snapshots; + writable_directories=writable) do gate_root + mkdir(joinpath(gate_root, dirname(relative), "leftover")) + return + end + return + end + mktempdir() do root + relative = "test/conformance/n6/evidence/example.jsonl" + command = normalized_gate_command(String["python", "validator.py"], + String[relative], root) + @test command.dir == root + @test command.exec[end-1:end] == ["--gate", relative] + @test !isabspath(command.exec[end]) + absolute = joinpath(root, relative) + @test_throws ErrorException normalized_gate_command( + String["python", "validator.py"], String[absolute], root) + return + end + for oracle in oracles + mktempdir() do root + for name in oracles + mkpath(joinpath(root, "oracles", name)) + end + oracle_dir = joinpath(root, "oracles", oracle) + mktempdir() do outside + symlink(outside, joinpath(oracle_dir, "build")) + @test_throws ErrorException intended_artifacts(root) + return + end + return + end + end + for oracle in ("arrow-rs", "parquet-java") + mktempdir() do root + for name in oracles + mkpath(joinpath(root, "oracles", name)) + end + build = joinpath(root, "oracles", oracle, "build") + mkpath(build) + mktempdir() do outside + symlink(outside, joinpath(build, "nested")) + @test_throws ErrorException intended_artifacts(root) + return + end + return + end + end + mktempdir() do root + for name in oracles + mkpath(joinpath(root, "oracles", name)) + end + mktempdir() do outside + symlink(outside, joinpath(root, "unlisted")) + @test_throws ErrorException intended_artifacts(root) + return + end + return + end + mktempdir() do root + site_packages = joinpath(root, "lib", "python3.12", "site-packages") + cache = joinpath(root, "lib", "python3.12", "encodings", + "__pycache__") + mkpath(site_packages) + mkpath(cache) + write(joinpath(site_packages, "ambient.py"), "forbidden\n") + write(joinpath(cache, "aliases.cpython-312.pyc"), "cache\n") + prune_interop_python!(root) + @test !ispath(site_packages) + @test !ispath(cache) + module_file = joinpath(root, "lib", "python3.12", "module.py") + write(module_file, "value = 1\n") + executable = joinpath(root, "bin", "python3.12") + mkpath(dirname(executable)) + write(executable, "runtime\n") + lock_interop_python!(root) + @test (stat(root).mode & 0o222) == 0 + @test (stat(module_file).mode & 0o222) == 0 + unlock_interop_python!(root) + @test (stat(root).mode & 0o200) != 0 + @test (stat(module_file).mode & 0o200) != 0 + return + end + mktempdir() do root + base = joinpath(root, "lib", "python3.12") + mkpath(base) + mktempdir() do outside + symlink(outside, joinpath(base, "site-packages")) + @test_throws ErrorException prune_interop_python!(root) + @test isdir(outside) + return + end + return + end + for ancestor in ("lib", joinpath("lib", "python3.12")) + mktempdir() do root + mktempdir() do outside + destination = joinpath(root, ancestor) + mkpath(dirname(destination)) + mkpath(joinpath(outside, "site-packages")) + sentinel = joinpath(outside, "site-packages", "sentinel") + write(sentinel, "preserve\n") + symlink(outside, destination) + @test_throws ErrorException prune_interop_python!(root) + @test read(sentinel, String) == "preserve\n" + return + end + return + end + end + mktempdir() do root + write(joinpath(root, "payload"), "data") + write(joinpath(root, "empty"), "") + symlink("payload", joinpath(root, "alias")) + inventory = validate_bounded_tree(root; max_entries=3, + max_file_bytes=4, max_total_bytes=4) + @test inventory.entries == 3 + @test inventory.bytes == 4 + @test occursin(SHA256_PATTERN, inventory.sha256) + @test_throws ErrorException validate_bounded_tree(root; + max_entries=2, max_file_bytes=4, max_total_bytes=4) + @test_throws ErrorException validate_bounded_tree(root; + max_entries=3, max_file_bytes=3, max_total_bytes=3) + return + end + mktempdir() do root + mktempdir() do outside + write(joinpath(outside, "sentinel"), "preserve\n") + symlink(joinpath(outside, "sentinel"), joinpath(root, "escape")) + @test_throws ErrorException validate_bounded_tree(root; + max_entries=1, max_file_bytes=16, max_total_bytes=16) + @test read(joinpath(outside, "sentinel"), String) == "preserve\n" + return + end + return + end + marker = "n6-test-ambient-secret-do-not-inherit" + withenv("N6_TEST_AMBIENT_SECRET" => marker) do + output = read(isolated_command(`/usr/bin/env`), String) + @test !occursin("N6_TEST_AMBIENT_SECRET", output) + @test !occursin(marker, output) + try + run(isolated_command(`/usr/bin/false`)) + @test false + catch error + message = sprint(showerror, error) + @test !occursin("N6_TEST_AMBIENT_SECRET", message) + @test !occursin(marker, message) + end + return + end + return +end + +include("julia/producer_gate.jl") + +@testset "N6 Parquet.jl producer identity helpers" begin + test_parquet_jl_identity_helpers() +end + +function test_preproduction_evidence() + manifest_snapshot = file_snapshot("test/conformance/n6/manifest.toml") + manifest = parse_toml_snapshot(manifest_snapshot) + snapshots = load_control_snapshots(manifest) + snapshots["test/conformance/n6/manifest.toml"] = manifest_snapshot + capabilities = parse_toml_snapshot(control_snapshot(snapshots, + manifest["capabilities_file"])) + fixtures = parse_toml_snapshot(control_snapshot(snapshots, + manifest["fixture_manifest_file"])) + load_generated_snapshots!(snapshots, fixtures) + load_evidence_snapshots!(snapshots, manifest) + model_cases = parse_toml_snapshot(control_snapshot(snapshots, + "test/conformance/n6/model/cases.toml")) + corpus_lines = collect(eachline(IOBuffer(control_snapshot(snapshots, + manifest["corpus_manifest_file"]).payload))) + capability_ids = Set{String}() + preflight = @testset "N6 fail-closed preflight" begin + validate_manifest(manifest) + capability_ids = validate_capabilities(capabilities, fixtures, + model_cases, manifest) + validate_fixtures(fixtures, model_cases, + corpus_lines, capability_ids, manifest) + validate_static_files(manifest, capabilities, fixtures, snapshots) + validate_running_julia(manifest) + end + isempty(Test.filter_errors(preflight)) || + error("N6 static preflight failed; external execution is disabled") + if get(ENV, "PARQUET_N6_GATE", "0") == "1" + require_gate(Sys.isapple(), "N6 gate requires macOS") + require_gate(Sys.ARCH == :aarch64, "N6 gate requires arm64") + require_gate(VERSION == v"1.12.6", + "N6 external gate requires Julia 1.12.6") + require_gate(startswith(strip(read(isolated_command( + `/usr/bin/sw_vers -productVersion`), String)), "15."), + "N6 gate requires macOS 15") + load_oracle_build_snapshots!(snapshots, manifest) + validate_source_roots(manifest, fixtures) + validate_python_gate(manifest, capabilities, fixtures, snapshots) + end + return +end + +@testset "N6 path safety helpers" begin + test_path_safety_helpers() +end + +@testset "N6 preproduction evidence" begin + test_preproduction_evidence() +end + +include("model/runtests.jl") diff --git a/test/conformance/n6/toolchains/duckdb.toml b/test/conformance/n6/toolchains/duckdb.toml new file mode 100644 index 0000000..85a9b24 --- /dev/null +++ b/test/conformance/n6/toolchains/duckdb.toml @@ -0,0 +1,23 @@ +descriptor_version = 1 +id = "duckdb-cp312-macos-15-arm64" +status = "verified" +authority = "duckdb" +platform = "macos-15-arm64" +python_version = "3.12.8" +python_source_revision = "2dc476bcb9142cd25d7e1d52392b73a3dcdf1756" +python_distribution_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250115/cpython-3.12.8%2B20250115-aarch64-apple-darwin-install_only_stripped.tar.gz" +python_distribution_sha256 = "dfb8a4c87116538717105ef3dec3668ae07590a5b5532109fec3ccad90be2fbc" +python_tree_policy = "extract-strip-site-packages-bytecode-v1" +python_executable_sha256 = "d6b64f766d3b08326aa10cdb37c9d922e3af38b57ec07caa28b894e5fccf6e69" +python_tree_sha256 = "e3b7dcdffba67f605b0fa3318656387e8d4265d34531c2bdbc43d3aba4a033ec" +wheels = [ + { name = "duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", sha256 = "f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae" }, +] +harness_status = "verified" +harness_file = "test/conformance/n6/harnesses/duckdb.py" +harness_sha256 = "96e9219d8c5c0d24d348280dcbad3ee2006a5ec6f3096bb7ca671592ef422754" +support_files = [ + { file = "test/conformance/n6/harnesses/common.py", sha256 = "8b48613a795deed37089f6dbfca8eb6ed18062ebfd907a4606c91ef3308776ad" }, +] +test_file = "test/conformance/n6/harnesses/test_common.py" +test_sha256 = "f3d90b023c5ec3e71713d96631054a218e4656f2345fe407daeff60ece3ae694" diff --git a/test/conformance/n6/toolchains/pyarrow.toml b/test/conformance/n6/toolchains/pyarrow.toml new file mode 100644 index 0000000..47d0ec3 --- /dev/null +++ b/test/conformance/n6/toolchains/pyarrow.toml @@ -0,0 +1,23 @@ +descriptor_version = 1 +id = "pyarrow-cp312-macos-15-arm64" +status = "verified" +authority = "pyarrow" +platform = "macos-15-arm64" +python_version = "3.12.8" +python_source_revision = "2dc476bcb9142cd25d7e1d52392b73a3dcdf1756" +python_distribution_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250115/cpython-3.12.8%2B20250115-aarch64-apple-darwin-install_only_stripped.tar.gz" +python_distribution_sha256 = "dfb8a4c87116538717105ef3dec3668ae07590a5b5532109fec3ccad90be2fbc" +python_tree_policy = "extract-strip-site-packages-bytecode-v1" +python_executable_sha256 = "d6b64f766d3b08326aa10cdb37c9d922e3af38b57ec07caa28b894e5fccf6e69" +python_tree_sha256 = "e3b7dcdffba67f605b0fa3318656387e8d4265d34531c2bdbc43d3aba4a033ec" +wheels = [ + { name = "pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", sha256 = "df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9" }, +] +harness_status = "verified" +harness_file = "test/conformance/n6/harnesses/pyarrow.py" +harness_sha256 = "6b8d6f2bd1dfe4ce26a3f772974dd5a3b875c17972264563cee8cfec49b748f0" +support_files = [ + { file = "test/conformance/n6/harnesses/common.py", sha256 = "8b48613a795deed37089f6dbfca8eb6ed18062ebfd907a4606c91ef3308776ad" }, +] +test_file = "test/conformance/n6/harnesses/test_common.py" +test_sha256 = "f3d90b023c5ec3e71713d96631054a218e4656f2345fe407daeff60ece3ae694" diff --git a/test/conformance/n6/validate_evidence.py b/test/conformance/n6/validate_evidence.py new file mode 100644 index 0000000..5839e4d --- /dev/null +++ b/test/conformance/n6/validate_evidence.py @@ -0,0 +1,2013 @@ +#!/usr/bin/env python3 +import argparse +import copy +import hashlib +import io +import json +import os +import pathlib +import re +import stat +import sys +import tempfile +import tomllib + +import jsonschema + +INT64_MIN = -(1 << 63) +INT64_MAX = (1 << 63) - 1 +INT32_MAX = (1 << 31) - 1 +STATISTIC_FIELDS = ( + "deprecated_min_hex", + "deprecated_max_hex", + "min_value_hex", + "max_value_hex", + "is_min_value_exact", + "is_max_value_exact", + "null_count", + "distinct_count", + "nan_count", +) +COLUMN_EVIDENCE_CAPABILITIES = { + "wire.column-order.type", + "wire.column-order.ieee", + "wire.column-order.empty", + "wire.statistics.deprecated-bounds", + "wire.statistics.modern-bounds", + "wire.statistics.exactness", + "wire.statistics.counts", + "wire.statistics.nan-count", + "semantic.type-order", + "semantic.ieee-total-order", + "semantic.logical-order", + "semantic.count-state", + "semantic.producer-trust", + "write.type-order", + "write.statistics-disabled", + "write.statistics-limit", + "compat.legacy-statistics", +} +COMPACT_TO_TTYPE = { + 1: 2, + 2: 2, + 3: 3, + 4: 6, + 5: 8, + 6: 10, + 7: 4, + 8: 11, + 9: 15, + 10: 14, + 11: 13, + 12: 12, + 13: 16, +} +CONVERTED_LOGICAL = { + "UTF8": "STRING", + "ENUM": "ENUM", + "DECIMAL": "DECIMAL", + "DATE": "DATE", + "TIME_MILLIS": "TIME", + "TIME_MICROS": "TIME", + "TIMESTAMP_MILLIS": "TIMESTAMP", + "TIMESTAMP_MICROS": "TIMESTAMP", + "UINT_8": "INTEGER", + "UINT_16": "INTEGER", + "UINT_32": "INTEGER", + "UINT_64": "INTEGER", + "INT_8": "INTEGER", + "INT_16": "INTEGER", + "INT_32": "INTEGER", + "INT_64": "INTEGER", + "JSON": "JSON", + "BSON": "BSON", + "INTERVAL": "INTERVAL", +} +GROUP_CONVERTED_TYPES = {"MAP", "MAP_KEY_VALUE", "LIST"} +INTEGER_CONVERTED = { + "UINT_8": (8, False), + "UINT_16": (16, False), + "UINT_32": (32, False), + "UINT_64": (64, False), + "INT_8": (8, True), + "INT_16": (16, True), + "INT_32": (32, True), + "INT_64": (64, True), +} +TIME_CONVERTED = { + "TIME_MILLIS": "MILLIS", + "TIME_MICROS": "MICROS", + "TIMESTAMP_MILLIS": "MILLIS", + "TIMESTAMP_MICROS": "MICROS", +} + + +class EvidenceError(ValueError): + pass + + +CONTROL_FILE_LIMIT = 4 * 1024 * 1024 + + +def metadata_identity(metadata): + return (metadata.st_dev, metadata.st_ino, metadata.st_mode, + metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, + metadata.st_ctime_ns) + + +def stable_regular_bytes(path, maximum_bytes, label, *, allow_empty=True): + path = pathlib.Path(path) + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise EvidenceError(f"{label} is not a regular file") + minimum = 0 if allow_empty else 1 + if not minimum <= metadata.st_size <= maximum_bytes: + raise EvidenceError(f"{label} has an invalid byte size") + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as error: + raise EvidenceError(f"cannot open {label} safely") from error + with os.fdopen(descriptor, "rb") as stream: + opened = os.fstat(stream.fileno()) + if metadata_identity(opened) != metadata_identity(metadata): + raise EvidenceError(f"{label} changed while it was opened") + value = stream.read(maximum_bytes + 1) + if len(value) > maximum_bytes or len(value) != opened.st_size or \ + stream.read(1): + raise EvidenceError(f"{label} changed size while it was read") + final = os.fstat(stream.fileno()) + if metadata_identity(final) != metadata_identity(opened): + raise EvidenceError(f"{label} changed while it was read") + current = path.lstat() + if metadata_identity(current) != metadata_identity(opened): + raise EvidenceError(f"{label} path changed while it was read") + return value + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def safe_relative(value): + if not isinstance(value, str) or not value or "\\" in value: + return False + if any(not character.isascii() or not ( + character.isalnum() or character in "._+@=-/") + for character in value): + return False + path = pathlib.PurePosixPath(value) + return not path.is_absolute() and all(part not in ("", ".", "..") + for part in path.parts) and path.as_posix() == value + + +def safe_evidence_file(value): + prefix = "test/conformance/n6/evidence/" + return safe_relative(value) and value.startswith(prefix) and \ + len(value) > len(prefix) and pathlib.PurePosixPath(value).name.endswith( + ".jsonl") and pathlib.PurePosixPath(value).name != ".jsonl" + + +def manifest_evidence_entry(manifest, evidence_id): + matches = [] + for section in ("frozen_evidence", "planned_evidence"): + matches.extend(entry for entry in manifest.get(section, []) + if entry["id"] == evidence_id) + if len(matches) != 1: + raise EvidenceError( + f"manifest evidence entry is absent or ambiguous: {evidence_id}") + return matches[0] + + +def repository_root(manifest_path): + manifest_file = pathlib.Path(manifest_path).resolve(strict=True) + try: + root = manifest_file.parents[3] + except IndexError as error: + raise EvidenceError("manifest path is outside the N6 repository layout") from error + if root.joinpath("test", "conformance", "n6", "manifest.toml") != manifest_file: + raise EvidenceError("manifest path is outside the canonical N6 location") + return root + + +def manifest_input_file(root, relative): + if not safe_relative(relative): + raise EvidenceError(f"manifest input path is unsafe: {relative}") + candidate = root.joinpath(*pathlib.PurePosixPath(relative).parts) + if not candidate.is_file(): + raise EvidenceError(f"manifest input file is absent: {relative}") + current = root + for part in pathlib.PurePosixPath(relative).parts: + current = current / part + if current.is_symlink(): + raise EvidenceError(f"manifest input contains a symbolic link: {relative}") + try: + candidate.resolve(strict=True).relative_to(root) + except ValueError as error: + raise EvidenceError(f"manifest input escapes the repository: {relative}") from error + return candidate + + +def toolchain_status(manifest, digest): + owners = [toolchain["status"] for toolchain in manifest["toolchain"] + if any(artifact["sha256"] == digest for artifact in toolchain["artifacts"])] + if len(owners) != 1: + raise EvidenceError("producer toolchain digest has ambiguous manifest ownership") + return owners[0] + + +def load_semantic_cases(root, manifest, capabilities, fixtures): + relative = "test/conformance/n6/model/cases.toml" + entries = [item for item in manifest["frozen_model"] + if item["file"] == relative] + if len(entries) != 1: + raise EvidenceError("semantic case manifest is absent or ambiguous") + path = manifest_input_file(root, relative) + value = stable_regular_bytes(path, CONTROL_FILE_LIMIT, + "N6 semantic case manifest", allow_empty=False) + if sha256_bytes(value) != entries[0]["sha256"]: + raise EvidenceError("semantic case manifest hash differs") + try: + model_cases = tomllib.loads(value.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise EvidenceError("semantic case manifest is invalid TOML") from error + if model_cases.get("schema_version") != 1 or \ + not isinstance(model_cases.get("case_groups"), list): + raise EvidenceError("semantic case manifest header differs") + capability_ids = {item["id"] for item in capabilities["capability"]} + contracts = set(fixtures["digest_contracts"]) + result = [] + identifiers = set() + for case in model_cases["case_groups"]: + required = {"id", "requirements", "capabilities", "digest_contract", + "expected_sha256"} + if not isinstance(case, dict) or set(case) != required: + raise EvidenceError("semantic case fields differ") + identifier = case["id"] + if not isinstance(identifier, str) or \ + re.fullmatch(r"[a-z0-9]+(?:[._-][a-z0-9]+)*", identifier) is None or \ + identifier in identifiers: + raise EvidenceError("semantic case ID is invalid or duplicated") + identifiers.add(identifier) + requirements = case["requirements"] + case_capabilities = case["capabilities"] + expected = case["expected_sha256"] + if not isinstance(requirements, list) or not requirements or \ + any(not isinstance(item, str) or not item for item in requirements): + raise EvidenceError(f"semantic case requirements differ: {identifier}") + if not isinstance(case_capabilities, list) or \ + case_capabilities != sorted(set(case_capabilities)) or \ + any(item not in capability_ids for item in case_capabilities): + raise EvidenceError(f"semantic case capabilities differ: {identifier}") + if case["digest_contract"] not in contracts or \ + not isinstance(expected, dict) or \ + set(expected) != set(case_capabilities) or \ + any(not isinstance(digest, str) or + re.fullmatch(r"[0-9a-f]{64}", digest) is None + for digest in expected.values()): + raise EvidenceError(f"semantic case digests differ: {identifier}") + result.append(case) + claimed = {} + fixture_ids = {item["id"] for item in + fixtures["fixture"] + fixtures["generated_case"]} + for authority in capabilities["authority"]: + for claim in authority.get("claim", []): + for case_id in claim["cases"]: + if case_id in fixture_ids: + continue + key = (case_id, claim["capability"]) + claimed.setdefault(key, set()).add(authority["id"]) + declared = {(case["id"], capability) + for case in result for capability in case["capabilities"]} + if set(claimed) != declared: + raise EvidenceError("semantic case capability scope differs from claims") + return result + + +def validate_model_producer_binding(root, manifest, capabilities): + expected_relative = \ + "test/conformance/n6/normalizers/model-producer.toml" + relative = manifest.get("model_producer_descriptor_file") + expected_sha256 = manifest.get("model_producer_descriptor_sha256") + if relative != expected_relative or not isinstance(expected_sha256, str): + raise EvidenceError("model producer descriptor declaration differs") + descriptor_path = manifest_input_file(root, relative) + payload = stable_regular_bytes(descriptor_path, CONTROL_FILE_LIMIT, + "N6 model producer descriptor", allow_empty=False) + descriptor_sha256 = sha256_bytes(payload) + if descriptor_sha256 != expected_sha256: + raise EvidenceError("model producer descriptor hash differs") + try: + descriptor = tomllib.loads(payload.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError) as error: + raise EvidenceError("model producer descriptor is invalid TOML") from error + if set(descriptor) != {"descriptor_version", "producer", "file"} or \ + descriptor["descriptor_version"] != 1 or \ + descriptor["producer"] != "n6-independent-model" or \ + not isinstance(descriptor["file"], list): + raise EvidenceError("model producer descriptor header differs") + expected_files = { + "test/conformance/n6/model/N6StatisticsModel.jl", + "test/conformance/n6/model/README.md", + "test/conformance/n6/model/cases.toml", + "test/conformance/n6/model/runtests.jl", + "test/conformance/n6/normalizers/common.py", + "test/conformance/n6/normalizers/model_bridge.jl", + "test/conformance/n6/normalizers/normalize_model.py", + "test/conformance/n6/normalizers/normalize_raw.py", + } + producer_hashes = {} + for item in descriptor["file"]: + if not isinstance(item, dict) or set(item) != {"path", "sha256"} or \ + not isinstance(item["path"], str) or \ + not isinstance(item["sha256"], str): + raise EvidenceError("model producer file entry differs") + file_path = manifest_input_file(root, item["path"]) + file_bytes = stable_regular_bytes(file_path, 32 * 1024 * 1024, + "N6 model producer file", allow_empty=False) + if sha256_bytes(file_bytes) != item["sha256"]: + raise EvidenceError( + f"model producer file hash differs: {item['path']}") + if item["path"] in producer_hashes: + raise EvidenceError("model producer file is duplicated") + producer_hashes[item["path"]] = item["sha256"] + if set(producer_hashes) != expected_files: + raise EvidenceError("model producer file set differs") + frozen_models = {item["file"]: item["sha256"] + for item in manifest["frozen_model"]} + if len(frozen_models) != len(manifest["frozen_model"]) or any( + producer_hashes.get(path) != digest + for path, digest in frozen_models.items()): + raise EvidenceError("model producer and frozen model differ") + authorities = [item for item in capabilities["authority"] + if item["id"] == "n6-independent-model"] + if len(authorities) != 1 or \ + authorities[0]["revision"] != descriptor_sha256: + raise EvidenceError("model authority does not bind its producer") + return None + + +def evidence_limits(manifest, fixtures, capabilities): + limits = manifest["evidence_limits"] + required = { + "max_inputs", "max_file_bytes", "max_total_bytes", "max_line_bytes", + "max_records_per_input", "max_records_total", + } + if set(limits) != required or any(type(limits[name]) is not int + for name in required): + raise EvidenceError("manifest evidence limits have invalid keys or types") + if not (0 < limits["max_inputs"] <= len(capabilities["authority"])): + raise EvidenceError("manifest evidence input limit is invalid") + if not (0 < limits["max_line_bytes"] <= limits["max_file_bytes"] <= + limits["max_total_bytes"]): + raise EvidenceError("manifest evidence byte limits are invalid") + maximum_records = 1 + sum(case["normalized_record_count"] + + len(case["capabilities"]) for case in + fixtures["fixture"] + fixtures["generated_case"]) + maximum_records += sum(len(case["capabilities"]) + for case in fixtures["semantic_case"]) + if limits["max_records_per_input"] != maximum_records: + raise EvidenceError("manifest per-input record limit is not fixture-derived") + if limits["max_records_total"] != maximum_records * limits["max_inputs"]: + raise EvidenceError("manifest total record limit is not input-derived") + return limits + + +def parse_int64(value, name): + parsed = int(value) + if parsed < INT64_MIN or parsed > INT64_MAX: + raise EvidenceError(f"{name} is outside signed Int64") + return parsed + + +def fixed_decimal_precision(type_length): + # This is the Parquet formula floor(log10(2^(8*n - 1) - 1)). The fixed + # decimal constant has enough digits to be exact over the signed i32 width + # accepted by the pinned IDL. + log10_2 = int( + "3010299956639811952137388947244930267681898814621085413104274611271081892744") + scale = 10 ** 76 + return ((8 * type_length - 1) * log10_2) // scale + + +def validate_leaf_annotations(leaf): + physical = leaf["physical_type"] + logical = leaf["logical_type"] + converted = leaf["converted_type"] + type_length = leaf["type_length"] + bit_width = leaf["bit_width"] + is_signed = leaf["is_signed"] + time_unit = leaf["time_unit"] + if type_length is not None and not -(1 << 31) <= type_length <= INT32_MAX: + raise EvidenceError("leaf has an invalid signed-i32 type length") + if physical == "FIXED_LEN_BYTE_ARRAY" and ( + type_length is None or type_length <= 0): + raise EvidenceError("fixed leaf lacks a positive signed-i32 type length") + if logical in ("MAP", "LIST", "VARIANT"): + raise EvidenceError(f"group logical type {logical} is present on a leaf") + if converted in GROUP_CONVERTED_TYPES: + raise EvidenceError(f"group converted type {converted} is present on a leaf") + if converted is not None: + expected_logical = CONVERTED_LOGICAL.get(converted) + if expected_logical is None or logical != expected_logical: + raise EvidenceError("logical and converted types are inconsistent") + + required_converted = { + "STRING": "UTF8", + "ENUM": "ENUM", + "DECIMAL": "DECIMAL", + "DATE": "DATE", + "JSON": "JSON", + "BSON": "BSON", + "INTERVAL": "INTERVAL", + }.get(logical) + if logical == "INTEGER" and bit_width is not None and is_signed is not None: + required_converted = ("INT_" if is_signed else "UINT_") + str(bit_width) + if logical in ("TIME", "TIMESTAMP") and time_unit in ("MILLIS", "MICROS"): + required_converted = logical + "_" + time_unit + if converted != required_converted: + raise EvidenceError("logical type lacks its exact compatible converted type") + return None + + +def validate_decimal_leaf(leaf): + physical = leaf["physical_type"] + logical = leaf["logical_type"] + type_length = leaf["type_length"] + precision = leaf["precision"] + scale = leaf["scale"] + decimal = logical == "DECIMAL" + if decimal: + if precision is None or scale is None or scale > precision: + raise EvidenceError("DECIMAL lacks valid precision and scale") + if physical == "INT32": + maximum_precision = 9 + elif physical == "INT64": + maximum_precision = 18 + elif physical == "FIXED_LEN_BYTE_ARRAY": + maximum_precision = fixed_decimal_precision(type_length) + elif physical == "BYTE_ARRAY": + maximum_precision = None + else: + raise EvidenceError("DECIMAL has an invalid physical type") + if maximum_precision is not None and precision > maximum_precision: + raise EvidenceError("DECIMAL precision exceeds its physical type") + elif precision is not None or scale is not None: + raise EvidenceError("non-DECIMAL leaf carries decimal parameters") + return None + + +def validate_integer_leaf(leaf): + physical = leaf["physical_type"] + logical = leaf["logical_type"] + converted = leaf["converted_type"] + bit_width = leaf["bit_width"] + is_signed = leaf["is_signed"] + integer = logical == "INTEGER" + if integer: + if bit_width is None or is_signed is None: + raise EvidenceError("INTEGER lacks width or signedness") + expected_physical = "INT64" if bit_width == 64 else "INT32" + if physical != expected_physical: + raise EvidenceError("INTEGER width contradicts its physical type") + if converted in INTEGER_CONVERTED and ( + bit_width, is_signed) != INTEGER_CONVERTED[converted]: + raise EvidenceError("INTEGER parameters contradict its converted type") + elif bit_width is not None or is_signed is not None: + raise EvidenceError("non-INTEGER leaf carries integer parameters") + return None + + +def validate_temporal_leaf(leaf): + physical = leaf["physical_type"] + logical = leaf["logical_type"] + converted = leaf["converted_type"] + time_unit = leaf["time_unit"] + adjusted = leaf["is_adjusted_to_utc"] + temporal = logical in ("TIME", "TIMESTAMP") + if temporal: + if time_unit is None or adjusted is None: + raise EvidenceError("temporal leaf lacks unit or UTC adjustment") + if logical == "TIME": + expected_physical = "INT32" if time_unit == "MILLIS" else "INT64" + else: + expected_physical = "INT64" + if physical != expected_physical: + raise EvidenceError("temporal unit contradicts its physical type") + if converted in TIME_CONVERTED: + if time_unit != TIME_CONVERTED[converted]: + raise EvidenceError("temporal parameters contradict its converted type") + elif time_unit is not None or adjusted is not None: + raise EvidenceError("non-temporal leaf carries temporal parameters") + return None + + +def validate_leaf_physical_type(leaf): + physical = leaf["physical_type"] + logical = leaf["logical_type"] + converted = leaf["converted_type"] + type_length = leaf["type_length"] + physical_by_logical = { + "STRING": "BYTE_ARRAY", + "ENUM": "BYTE_ARRAY", + "DATE": "INT32", + "JSON": "BYTE_ARRAY", + "BSON": "BYTE_ARRAY", + "UUID": "FIXED_LEN_BYTE_ARRAY", + "FLOAT16": "FIXED_LEN_BYTE_ARRAY", + "INTERVAL": "FIXED_LEN_BYTE_ARRAY", + "GEOMETRY": "BYTE_ARRAY", + "GEOGRAPHY": "BYTE_ARRAY", + } + expected_physical = physical_by_logical.get(logical) + if expected_physical is not None and physical != expected_physical: + raise EvidenceError(f"{logical} has an invalid physical type") + expected_lengths = {"UUID": 16, "FLOAT16": 2, "INTERVAL": 12} + if logical in expected_lengths and type_length != expected_lengths[logical]: + raise EvidenceError(f"{logical} has an invalid fixed width") + if logical == "INTERVAL" and converted != "INTERVAL": + raise EvidenceError("INTERVAL lacks its required converted type") + if logical == "NONE" and converted is not None: + raise EvidenceError("NONE logical type carries a converted annotation") + return None + + +def validate_geospatial_leaf(leaf): + logical = leaf["logical_type"] + crs = leaf["crs"] + geography_algorithm = leaf["geography_algorithm"] + geospatial = logical in ("GEOMETRY", "GEOGRAPHY") + if geospatial: + if logical == "GEOMETRY" and geography_algorithm is not None: + raise EvidenceError("GEOMETRY carries a geography algorithm") + elif crs is not None or geography_algorithm is not None: + raise EvidenceError("non-geospatial leaf carries geospatial parameters") + return None + + +def validate_leaf_schema(leaf): + validate_leaf_annotations(leaf) + validate_decimal_leaf(leaf) + validate_integer_leaf(leaf) + validate_temporal_leaf(leaf) + validate_leaf_physical_type(leaf) + validate_geospatial_leaf(leaf) + return None + + +def authority_map(capabilities): + result = {} + for authority in capabilities["authority"]: + if authority["id"] in result: + raise EvidenceError(f"duplicate authority {authority['id']}") + result[authority["id"]] = authority + return result + + +def case_map(fixtures): + result = {} + for fixture in fixtures["fixture"]: + if fixture["id"] in result: + raise EvidenceError(f"duplicate fixture {fixture['id']}") + expected = 1 + fixture["row_group_count"] * fixture["leaf_count"] + if fixture["normalized_record_count"] != expected: + raise EvidenceError(f"invalid normalized record count for {fixture['id']}") + result[fixture["id"]] = dict(fixture, generated=False, semantic=False, + digest_contract=fixtures["default_digest_contract"]) + for generated in fixtures["generated_case"]: + if generated["id"] in result: + raise EvidenceError(f"duplicate case {generated['id']}") + expected = 1 + generated["row_group_count"] * generated["leaf_count"] + if generated["normalized_record_count"] != expected: + raise EvidenceError(f"invalid normalized record count for {generated['id']}") + result[generated["id"]] = dict(generated, generated=True, semantic=False, + file=generated["output_file"]) + for semantic in fixtures["semantic_case"]: + if semantic["id"] in result: + raise EvidenceError(f"duplicate semantic case {semantic['id']}") + result[semantic["id"]] = dict(semantic, generated=False, semantic=True) + return result + + +def claim_statuses(authority): + pairs = {} + for claim in authority.get("claim", []): + for case_id in claim["cases"]: + key = (case_id, claim["capability"]) + if key in pairs: + raise EvidenceError(f"duplicate authority claim {key}") + pairs[key] = claim["status"] + return pairs + + +def decode_compact_header(header): + raw = bytes.fromhex(header) + first = raw[0] + compact_type = first & 0x0F + delta = first >> 4 + if compact_type == 0: + if raw != b"\x00": + raise EvidenceError("STOP ColumnOrder header has trailing bytes") + return None, None + if compact_type not in COMPACT_TO_TTYPE: + raise EvidenceError("ColumnOrder header has an invalid Compact-Thrift type") + if delta: + if len(raw) != 1: + raise EvidenceError("delta ColumnOrder header has trailing bytes") + field_id = delta + else: + encoded = raw[1:] + if not encoded or len(encoded) > 3 or encoded[-1] & 0x80: + raise EvidenceError("ColumnOrder field ID varint is incomplete") + value = 0 + shift = 0 + for index, byte in enumerate(encoded): + if index < len(encoded) - 1 and not byte & 0x80: + raise EvidenceError("ColumnOrder field ID header has trailing bytes") + value |= (byte & 0x7F) << shift + shift += 7 + canonical = [] + remaining = value + while True: + byte = remaining & 0x7F + remaining >>= 7 + canonical.append(byte | (0x80 if remaining else 0)) + if not remaining: + break + if bytes(canonical) != encoded: + raise EvidenceError("ColumnOrder field ID uses a noncanonical varint") + field_id = (value >> 1) ^ -(value & 1) + if not -(1 << 15) <= field_id < (1 << 15): + raise EvidenceError("ColumnOrder field ID is outside signed Int16") + return field_id, COMPACT_TO_TTYPE[compact_type] + + +def validate_column_order(order): + state = order["state"] + field_id = order["field_id"] + wire_type = order["wire_type"] + header = order["header_hex"] + if state == "ABSENT": + if any(value is not None for value in (field_id, wire_type, header)): + raise EvidenceError("absent ColumnOrder carries raw fields") + return None + if header is None: + raise EvidenceError("present ColumnOrder lacks its raw header") + decoded_field, decoded_type = decode_compact_header(header) + if (field_id, wire_type) != (decoded_field, decoded_type): + raise EvidenceError("ColumnOrder raw header contradicts its decoded fields") + if state == "TYPE_ORDER": + if (field_id, wire_type) != (1, 12): + raise EvidenceError("TYPE_ORDER has an invalid raw header") + elif state == "IEEE_754_TOTAL_ORDER": + if (field_id, wire_type) != (2, 12): + raise EvidenceError("IEEE order has an invalid raw header") + elif state == "UNKNOWN": + if field_id is None or field_id in (1, 2) or wire_type is None: + raise EvidenceError("unknown ColumnOrder is not preserved") + elif state == "WRONG_TYPE": + if field_id not in (1, 2) or wire_type in (None, 12): + raise EvidenceError("wrong-type ColumnOrder is inconsistent") + elif state == "EMPTY": + if field_id is not None or wire_type is not None or header != "00": + raise EvidenceError("empty ColumnOrder carries a member") + return None + + +def validate_column(record): + num_values = parse_int64(record["num_values"], "num_values") + if num_values < 0: + raise EvidenceError("num_values is negative") + counts = {} + for name in ("null_count", "distinct_count", "nan_count"): + value = record[name] + counts[name] = None if value is None else parse_int64(value, name) + if counts[name] is not None and not 0 <= counts[name] <= num_values: + raise EvidenceError(f"{name} is outside the column value count") + physical = record["leaf_schema"]["physical_type"] + logical = record["leaf_schema"]["logical_type"] + validate_leaf_schema(record["leaf_schema"]) + if counts["nan_count"] is not None and physical not in ("FLOAT", "DOUBLE") and logical != "FLOAT16": + raise EvidenceError("nan_count is present on a non-floating leaf") + if counts["null_count"] is not None and counts["nan_count"] is not None: + if counts["null_count"] + counts["nan_count"] > num_values: + raise EvidenceError("null_count plus nan_count exceeds num_values") + if counts["null_count"] is not None and counts["distinct_count"] is not None: + if counts["distinct_count"] > num_values - counts["null_count"]: + raise EvidenceError("distinct_count exceeds non-null values") + if not record["has_statistics"]: + if any(record[name] is not None for name in STATISTIC_FIELDS): + raise EvidenceError("statistics fields are present when has_statistics is false") + if record["unknown_statistics_field_ids"]: + raise EvidenceError("unknown statistics fields exist without Statistics") + unknown = record["unknown_statistics_field_ids"] + if unknown != sorted(unknown): + raise EvidenceError("unknown statistics field IDs are not sorted") + if any(field_id in range(1, 10) for field_id in unknown): + raise EvidenceError("known statistics field ID is marked unknown") + validate_column_order(record["column_order"]) + return None + + +def validate_record_schema(records, validator): + if not records: + raise EvidenceError("the first record must be run") + for record in records: + if not isinstance(record, dict): + raise EvidenceError("every evidence record must be a JSON object") + validator.validate(record) + if records[0]["record"] != "run": + raise EvidenceError("the first record must be run") + if sum(record["record"] == "run" for record in records) != 1: + raise EvidenceError("evidence must contain exactly one run record") + return None + + +def validate_run_record(run, manifest, capabilities, fixtures, input_hashes, + gate): + for field, expected in input_hashes["run"].items(): + if run[field] != expected: + raise EvidenceError(f"run {field} does not match the frozen input") + authorities = authority_map(capabilities) + if run["producer"] not in authorities: + raise EvidenceError(f"unknown producer {run['producer']}") + authority = authorities[run["producer"]] + if run["producer_version"] != authority["version"]: + raise EvidenceError("producer version does not match its authority") + if run["source_revision"] != authority["revision"]: + raise EvidenceError("producer revision does not match its authority") + if run["toolchain_sha256"] not in authority["toolchain_sha256"]: + raise EvidenceError("producer toolchain is not pinned by its authority") + selected_toolchain_status = toolchain_status(manifest, run["toolchain_sha256"]) + if gate and selected_toolchain_status != "verified": + raise EvidenceError("gate producer toolchain is not verified") + claims = claim_statuses(authority) + known_fixtures = case_map(fixtures) + return claims, known_fixtures + + +def validate_file_record(record, fixture, files, gate): + case_id = record["case_id"] + if fixture["semantic"]: + raise EvidenceError(f"semantic case has a file record: {case_id}") + if case_id in files: + raise EvidenceError(f"duplicate file record for {case_id}") + if record["file"] != fixture["file"]: + raise EvidenceError(f"file path mismatch for {case_id}") + if not fixture["generated"]: + if record["sha256"] != fixture["sha256"] or \ + record["size"] != fixture["size"]: + raise EvidenceError(f"file identity mismatch for {case_id}") + elif fixture["output_identity_status"] == "verified": + if record["sha256"] != fixture["output_sha256"] or \ + record["size"] != fixture["output_size"]: + raise EvidenceError(f"generated file identity mismatch for {case_id}") + elif gate: + raise EvidenceError(f"generated file identity is not verified: {case_id}") + if record["footer_length"] > record["size"] - 12: + raise EvidenceError(f"footer is not contained for {case_id}") + if record["row_group_count"] != fixture["row_group_count"]: + raise EvidenceError(f"row-group count mismatch for {case_id}") + if record["leaf_count"] != fixture["leaf_count"]: + raise EvidenceError(f"leaf count mismatch for {case_id}") + if record["column_order_count"] not in (None, record["leaf_count"]): + raise EvidenceError(f"column-order cardinality mismatch for {case_id}") + files[case_id] = record + return None + + +def validate_column_record(record, fixture, columns): + case_id = record["case_id"] + if fixture["semantic"]: + raise EvidenceError(f"semantic case has a column record: {case_id}") + key = (case_id, record["row_group"], record["leaf"]) + if key in columns: + raise EvidenceError(f"duplicate column record {key}") + if record["file"] != fixture["file"]: + raise EvidenceError(f"column file mismatch for {case_id}") + if record["row_group"] >= fixture["row_group_count"]: + raise EvidenceError(f"row-group ordinal is outside {case_id}") + if record["leaf"] >= fixture["leaf_count"]: + raise EvidenceError(f"leaf ordinal is outside {case_id}") + validate_column(record) + columns[key] = record + return None + + +def validate_case_result(record, fixture, results, claims, gate): + case_id = record["case_id"] + key = (case_id, record["capability_id"]) + if key in results: + raise EvidenceError(f"duplicate case result {key}") + if record["capability_id"] not in fixture["capabilities"]: + raise EvidenceError(f"capability is outside fixture scope: {key}") + if record["digest_contract"] != fixture["digest_contract"]: + raise EvidenceError(f"digest contract is outside fixture scope: {key}") + if key not in claims: + raise EvidenceError(f"producer has no claim for capability: {key}") + claim_status = claims[key] + if record["status"] == "PASS" and \ + record["expected_sha256"] != record["actual_sha256"]: + raise EvidenceError(f"passing digests differ for {key}") + if fixture["semantic"] and record["status"] == "PASS" and \ + record["expected_sha256"] != fixture["expected_sha256"][ + record["capability_id"]]: + raise EvidenceError(f"semantic case digest differs from its manifest: {key}") + if record["status"] == "PASS" and claim_status not in ("verified", "planned"): + raise EvidenceError(f"producer cannot pass its {claim_status} claim: {key}") + if record["status"] == "UNSUPPORTED" and claim_status != "unsupported": + raise EvidenceError(f"unsupported result is not reviewed: {key}") + if record["status"] == "FAIL" and claim_status not in ("verified", "planned"): + raise EvidenceError(f"producer cannot fail its {claim_status} claim: {key}") + if gate and record["status"] == "PASS" and claim_status != "verified": + raise EvidenceError(f"passing gate claim is not verified: {key}") + if gate and record["status"] == "FAIL": + raise EvidenceError(f"failed gate result: {key}") + results[key] = record + return None + + +def collect_evidence_records(records, known_fixtures, claims, gate): + files = {} + columns = {} + results = {} + for record in records[1:]: + kind = record["record"] + case_id = record["case_id"] + if case_id not in known_fixtures: + raise EvidenceError(f"unknown fixture {case_id}") + fixture = known_fixtures[case_id] + if kind == "file": + validate_file_record(record, fixture, files, gate) + elif kind == "column_statistics": + validate_column_record(record, fixture, columns) + elif kind == "case_result": + validate_case_result(record, fixture, results, claims, gate) + else: + raise EvidenceError(f"unsupported evidence record kind: {kind}") + return files, columns, results + + +def validate_case_columns(case_id, case_columns, file_record, fixture): + expected_pairs = { + (row_group, leaf) + for row_group in range(fixture["row_group_count"]) + for leaf in range(fixture["leaf_count"]) + } + actual_pairs = {(record["row_group"], record["leaf"]) + for record in case_columns} + if actual_pairs != expected_pairs: + raise EvidenceError(f"incomplete row-group and leaf coverage for {case_id}") + orders_present = file_record["column_order_count"] is not None + for record in case_columns: + order_absent = record["column_order"]["state"] == "ABSENT" + if orders_present == order_absent: + raise EvidenceError( + f"column-order vector presence contradicts leaf state for {case_id}") + leaf_facts = {} + leaf_orders = {} + for record in case_columns: + fact = (record["path"], record["leaf_schema"]) + previous = leaf_facts.setdefault(record["leaf"], fact) + if previous != fact: + raise EvidenceError(f"leaf schema changes across row groups for {case_id}") + previous_order = leaf_orders.setdefault(record["leaf"], + record["column_order"]) + if previous_order != record["column_order"]: + raise EvidenceError(f"column order changes across row groups for {case_id}") + paths = {tuple(fact[0]) for fact in leaf_facts.values()} + if len(paths) != fixture["leaf_count"]: + raise EvidenceError(f"leaf paths are not unique for {case_id}") + return None + + +def validate_record_coverage(run, files, columns, results, known_fixtures): + by_case = {} + for key, record in columns.items(): + by_case.setdefault(key[0], []).append(record) + for case_id, case_columns in by_case.items(): + if case_id not in files: + raise EvidenceError(f"columns have no file record for {case_id}") + validate_case_columns(case_id, case_columns, files[case_id], + known_fixtures[case_id]) + for case_id in files: + if case_id not in by_case and not any(key[0] == case_id for key in results): + raise EvidenceError(f"file record has no evidence for {case_id}") + for key, result in results.items(): + if key[0] not in files and not known_fixtures[key[0]]["semantic"]: + raise EvidenceError(f"case result has no file record: {key}") + unsupported_cases = { + case_id for (case_id, _), record in results.items() + if record["status"] == "UNSUPPORTED" + } + if run["unsupported_cases"] != sorted(unsupported_cases): + raise EvidenceError("run unsupported_cases does not match case results") + return by_case + + +def validate_records(records, validator, manifest, capabilities, fixtures, + input_hashes, gate=False): + validate_record_schema(records, validator) + run = records[0] + claims, known_fixtures = validate_run_record(run, manifest, capabilities, + fixtures, input_hashes, gate) + files, columns, results = collect_evidence_records(records, known_fixtures, + claims, gate) + by_case = validate_record_coverage(run, files, columns, results, + known_fixtures) + return { + "run": run, + "files": files, + "columns": columns, + "column_cases": set(by_case), + "results": results, + "passes": {key for key, record in results.items() + if record["status"] == "PASS"}, + } + + +def check_evidence_file_metadata(path, label, limits): + try: + metadata = path.stat(follow_symlinks=False) + except FileNotFoundError as error: + raise EvidenceError(f"evidence file is absent: {label}") from error + if not stat.S_ISREG(metadata.st_mode): + raise EvidenceError(f"evidence is not a regular file: {label}") + if not 0 < metadata.st_size <= limits["max_file_bytes"]: + raise EvidenceError(f"evidence has an invalid byte size: {label}") + return metadata.st_size + + +def checked_gate_input(relative, manifest, manifest_path, used_entries, limits): + if not safe_evidence_file(relative): + raise EvidenceError(f"gate evidence path is not canonical and relative: {relative}") + frozen = [entry for entry in manifest["frozen_evidence"] + if entry["file"] == relative] + planned = [entry for entry in manifest["planned_evidence"] + if entry["file"] == relative] + if planned: + raise EvidenceError(f"planned evidence cannot satisfy the gate: {relative}") + if len(frozen) != 1: + raise EvidenceError(f"gate evidence is undeclared or ambiguous: {relative}") + entry = frozen[0] + if entry["id"] in used_entries: + raise EvidenceError(f"duplicate gate evidence entry: {entry['id']}") + if entry["status"] != "verified" or entry["format"] != "normalized-jsonl": + raise EvidenceError(f"gate evidence is not verified normalized JSONL: {relative}") + if entry["schema_file"] != manifest["evidence_schema_file"] or \ + entry["fixture_manifest_file"] != manifest["fixture_manifest_file"]: + raise EvidenceError(f"gate evidence uses the wrong frozen inputs: {relative}") + if entry["schema_sha256"] != manifest["evidence_schema_sha256"]: + raise EvidenceError(f"gate evidence uses the wrong schema hash: {relative}") + root = repository_root(manifest_path) + candidate = root.joinpath(*pathlib.PurePosixPath(relative).parts) + current = root + for part in pathlib.PurePosixPath(relative).parts: + current = current / part + if current.is_symlink(): + raise EvidenceError(f"gate evidence path contains a symbolic link: {relative}") + try: + candidate.resolve(strict=True).relative_to(root) + except ValueError as error: + raise EvidenceError(f"gate evidence path escapes the repository: {relative}") from error + used_entries.add(entry["id"]) + return entry, candidate + + +def validate_gate_entry_coverage(manifest, used_entries): + expected_entries = [entry["id"] for entry in manifest["frozen_evidence"] + if entry["format"] == "normalized-jsonl"] + expected = set(expected_entries) + if len(expected) != len(expected_entries): + raise EvidenceError("normalized frozen evidence IDs are ambiguous") + if used_entries != expected: + missing = sorted(expected - used_entries) + extra = sorted(used_entries - expected) + raise EvidenceError( + "gate evidence set differs from normalized frozen evidence: " + f"missing={missing}, extra={extra}") + return None + + +def read_jsonl_bytes(value, label, limits): + if len(value) > limits["max_file_bytes"]: + raise EvidenceError(f"{label}: evidence file exceeds its byte limit") + + def unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise EvidenceError(f"{label}: duplicate JSON object key: {key}") + value[key] = item + return value + + def invalid_constant(value): + raise EvidenceError(f"{label}: invalid JSON numeric constant: {value}") + + def invalid_float(value): + raise EvidenceError(f"{label}: floating JSON number is forbidden: {value}") + + def canonical_integer(value): + if value != "0" and (not value or value[0] == "0" or + value.startswith("-0")): + raise EvidenceError(f"{label}: noncanonical JSON integer: {value}") + return int(value) + + records = [] + with io.BytesIO(value) as stream: + line_number = 0 + while True: + line = stream.readline(limits["max_line_bytes"] + 1) + if not line: + break + line_number += 1 + if len(line) > limits["max_line_bytes"]: + raise EvidenceError( + f"{label}:{line_number}: line exceeds its byte limit") + if not line.endswith(b"\n"): + raise EvidenceError(f"{label}:{line_number}: missing final newline") + decoded = line.decode("utf-8") + records.append(json.loads(decoded, object_pairs_hook=unique_object, + parse_constant=invalid_constant, parse_float=invalid_float, + parse_int=canonical_integer)) + if len(records) > limits["max_records_per_input"]: + raise EvidenceError( + f"{label}: evidence file exceeds its record limit") + return records + + +def read_jsonl(path, limits): + value = stable_regular_bytes(path, limits["max_file_bytes"], + f"evidence file {path}", allow_empty=False) + return read_jsonl_bytes(value, str(path), limits) + + +def validate_comparison_groups(fixtures, passing_results): + groups = {} + for case in fixtures["generated_case"]: + group = case["comparison_group"] + if group: + groups.setdefault(group, []).append(case) + for group, cases in groups.items(): + capabilities = set(cases[0]["capabilities"]) + if any(set(case["capabilities"]) != capabilities for case in cases): + raise EvidenceError(f"comparison group has inconsistent capabilities: {group}") + for capability in capabilities: + observations = [] + for case in cases: + records = passing_results.get((case["id"], capability), []) + if not records: + raise EvidenceError( + f"comparison group lacks a passing result: {(group, capability)}") + observations.extend((record["digest_contract"], + record["actual_sha256"]) for _, record in records) + if len(set(observations)) != 1: + raise EvidenceError( + f"comparison group results disagree: {(group, capability)}") + return None + + +def canonical_record(record): + return json.dumps(record, ensure_ascii=True, separators=(",", ":"), + sort_keys=True) + + +def validate_gate_bindings(args, manifest, input_hashes): + root = repository_root(args.manifest) + expected = { + "schema": manifest["evidence_schema_file"], + "capabilities": manifest["capabilities_file"], + "fixtures": manifest["fixture_manifest_file"], + } + for argument, relative in expected.items(): + supplied = pathlib.Path(getattr(args, argument)).resolve(strict=True) + frozen = root.joinpath(*pathlib.PurePosixPath(relative).parts) + if supplied != frozen: + raise EvidenceError(f"gate {argument} is not the manifest-pinned file") + expected_hashes = { + "schema": (input_hashes["run"]["evidence_schema_sha256"], + manifest["evidence_schema_sha256"]), + "capabilities": (input_hashes["run"]["capabilities_sha256"], + manifest["capabilities_sha256"]), + "fixtures": (input_hashes["run"]["fixture_manifest_sha256"], + manifest["fixture_manifest_sha256"]), + } + for argument, (actual_hash, expected_hash) in expected_hashes.items(): + if actual_hash != expected_hash: + raise EvidenceError(f"gate {argument} hash does not match the manifest") + direct_inputs = { + "plan": (input_hashes["run"]["plan_sha256"], + manifest["plan_sha256"]), + "corpus": (input_hashes["run"]["corpus_manifest_sha256"], + manifest["corpus_manifest_sha256"]), + "artifacts": (input_hashes["artifact_manifest_sha256"], + manifest["artifact_manifest_sha256"]), + } + for name, (actual_hash, expected_hash) in direct_inputs.items(): + if actual_hash != expected_hash: + raise EvidenceError(f"gate {name} hash does not match the manifest") + return root + + +def base_self_test(manifest, capability_data, fixture_data, input_hashes): + zero = "0" * 64 + producer = next(item for item in capability_data["authority"] + if item["id"] == "n6-raw-java") + if len(producer["toolchain_sha256"]) != 1: + raise EvidenceError("raw self-test requires one pinned toolchain") + fixture = next(item for item in fixture_data["fixture"] if item["id"] == "apache-binary") + run = { + "record": "run", + "schema_version": 2, + "evidence_id": "self-test", + "producer": "n6-raw-java", + "producer_version": producer["version"], + "source_revision": producer["revision"], + "plan_sha256": input_hashes["run"]["plan_sha256"], + "capabilities_sha256": input_hashes["run"]["capabilities_sha256"], + "fixture_manifest_sha256": + input_hashes["run"]["fixture_manifest_sha256"], + "corpus_manifest_sha256": + input_hashes["run"]["corpus_manifest_sha256"], + "evidence_schema_sha256": + input_hashes["run"]["evidence_schema_sha256"], + "toolchain_sha256": producer["toolchain_sha256"][0], + "unsupported_cases": [], + } + file_record = { + "record": "file", + "schema_version": 2, + "case_id": fixture["id"], + "file": fixture["file"], + "sha256": fixture["sha256"], + "size": fixture["size"], + "footer_length": 100, + "row_group_count": 1, + "leaf_count": 1, + "column_order_count": 1, + "created_by_present": False, + "created_by": None, + } + column = { + "record": "column_statistics", + "schema_version": 2, + "case_id": fixture["id"], + "file": fixture["file"], + "row_group": 0, + "leaf": 0, + "path": ["value"], + "leaf_schema": { + "physical_type": "BYTE_ARRAY", + "logical_type": "NONE", + "converted_type": None, + "type_length": None, + "precision": None, + "scale": None, + "bit_width": None, + "is_signed": None, + "time_unit": None, + "is_adjusted_to_utc": None, + "crs": None, + "geography_algorithm": None, + }, + "column_order": { + "state": "TYPE_ORDER", + "field_id": 1, + "wire_type": 12, + "header_hex": "1c", + }, + "num_values": "1", + "has_statistics": False, + "deprecated_min_hex": None, + "deprecated_max_hex": None, + "min_value_hex": None, + "max_value_hex": None, + "is_min_value_exact": None, + "is_max_value_exact": None, + "null_count": None, + "distinct_count": None, + "nan_count": None, + "unknown_statistics_field_ids": [], + } + result = { + "record": "case_result", + "schema_version": 2, + "case_id": fixture["id"], + "capability_id": "wire.column-order.type", + "digest_contract": fixture_data["default_digest_contract"], + "status": "PASS", + "expected_sha256": zero, + "actual_sha256": zero, + "detail": "self-test", + } + return [run, file_record, column, result] + + +def validate_self_test_records(records, context, gate=False): + return validate_records(records, *context, gate=gate) + + +def reject_self_test_records(records, context, gate=False): + try: + validate_self_test_records(records, context, gate) + except (EvidenceError, jsonschema.ValidationError, ValueError): + return None + raise EvidenceError("an adversarial self-test record was accepted") + + +def reject_self_test_mutation(base, context, mutator): + records = copy.deepcopy(base) + mutator(records) + reject_self_test_records(records, context) + return None + + +def accept_self_test_leaf(base, context, fields): + records = copy.deepcopy(base) + records[2]["leaf_schema"].update(fields) + validate_self_test_records(records, context) + return None + + +def reject_self_test_leaf(base, context, fields): + records = copy.deepcopy(base) + records[2]["leaf_schema"].update(fields) + reject_self_test_records(records, context) + return None + + +def self_test_compact_headers(): + header_cases = { + "00": (None, None), + "1c": (1, 12), + "2c": (2, 12), + "15": (1, 8), + "0cfeff03": (32767, 12), + "0cffff03": (-32768, 12), + } + for header, expected in header_cases.items(): + if decode_compact_header(header) != expected: + raise EvidenceError(f"Compact-Thrift header self-test failed: {header}") + for header in ("0e", "0c", "0c8200", "1c00"): + try: + decode_compact_header(header) + except EvidenceError: + pass + else: + raise EvidenceError(f"invalid Compact-Thrift header passed: {header}") + return None + + +def self_test_valid_leaves(base, context): + valid_leaves = [ + {"physical_type": "BYTE_ARRAY", "logical_type": "STRING", + "converted_type": "UTF8"}, + {"physical_type": "INT32", "logical_type": "DECIMAL", + "converted_type": "DECIMAL", "precision": 9, "scale": 2}, + {"physical_type": "INT32", "logical_type": "INTEGER", + "converted_type": "UINT_16", "bit_width": 16, + "is_signed": False}, + {"physical_type": "INT32", "logical_type": "TIME", + "converted_type": "TIME_MILLIS", "time_unit": "MILLIS", + "is_adjusted_to_utc": True}, + {"physical_type": "INT32", "logical_type": "TIME", + "converted_type": "TIME_MILLIS", "time_unit": "MILLIS", + "is_adjusted_to_utc": False}, + {"physical_type": "INT64", "logical_type": "TIMESTAMP", + "converted_type": "TIMESTAMP_MICROS", "time_unit": "MICROS", + "is_adjusted_to_utc": False}, + {"physical_type": "INT64", "logical_type": "TIMESTAMP", + "time_unit": "NANOS", "is_adjusted_to_utc": False}, + {"physical_type": "FIXED_LEN_BYTE_ARRAY", "logical_type": "UUID", + "type_length": 16}, + {"physical_type": "FIXED_LEN_BYTE_ARRAY", "logical_type": "FLOAT16", + "type_length": 2}, + {"physical_type": "FIXED_LEN_BYTE_ARRAY", "logical_type": "INTERVAL", + "converted_type": "INTERVAL", "type_length": 12}, + {"physical_type": "BYTE_ARRAY", "logical_type": "GEOGRAPHY", + "crs": "OGC:CRS84", "geography_algorithm": "SPHERICAL"}, + {"physical_type": "INT96", "logical_type": "UNKNOWN"}, + {"physical_type": "BYTE_ARRAY", "type_length": -1}, + ] + for fields in valid_leaves: + accept_self_test_leaf(base, context, fields) + return None + + +def self_test_multileaf_coverage(base, context, fixtures): + multi_fixture = next(item for item in fixtures["fixture"] + if item["id"] == "apache-alltypes-dictionary") + multi = [copy.deepcopy(base[0]), copy.deepcopy(base[1])] + multi[1].update({ + "case_id": multi_fixture["id"], + "file": multi_fixture["file"], + "sha256": multi_fixture["sha256"], + "size": multi_fixture["size"], + "row_group_count": multi_fixture["row_group_count"], + "leaf_count": multi_fixture["leaf_count"], + "column_order_count": None, + }) + for leaf in range(multi_fixture["leaf_count"]): + column = copy.deepcopy(base[2]) + column.update({ + "case_id": multi_fixture["id"], + "file": multi_fixture["file"], + "leaf": leaf, + "path": [f"value_{leaf}"], + }) + column["column_order"] = { + "state": "ABSENT", + "field_id": None, + "wire_type": None, + "header_hex": None, + } + multi.append(column) + validate_self_test_records(multi, context) + duplicate_path = copy.deepcopy(multi) + duplicate_path[3]["path"] = duplicate_path[2]["path"] + reject_self_test_records(duplicate_path, context) + return None + + +def self_test_multirow_coverage(base, context, fixtures): + multi_row_fixture = next(item for item in fixtures["fixture"] + if item["id"] == "apache-floating-orders-nan-count") + multi_row = [copy.deepcopy(base[0]), copy.deepcopy(base[1])] + multi_row[1].update({ + "case_id": multi_row_fixture["id"], + "file": multi_row_fixture["file"], + "sha256": multi_row_fixture["sha256"], + "size": multi_row_fixture["size"], + "row_group_count": multi_row_fixture["row_group_count"], + "leaf_count": multi_row_fixture["leaf_count"], + "column_order_count": multi_row_fixture["leaf_count"], + }) + for row_group in range(multi_row_fixture["row_group_count"]): + for leaf in range(multi_row_fixture["leaf_count"]): + column = copy.deepcopy(base[2]) + column.update({ + "case_id": multi_row_fixture["id"], + "file": multi_row_fixture["file"], + "row_group": row_group, + "leaf": leaf, + "path": [f"value_{leaf}"], + }) + multi_row.append(column) + validate_self_test_records(multi_row, context) + inconsistent_order = copy.deepcopy(multi_row) + inconsistent_order[-1]["column_order"] = { + "state": "IEEE_754_TOTAL_ORDER", + "field_id": 2, + "wire_type": 12, + "header_hex": "2c", + } + reject_self_test_records(inconsistent_order, context) + return None + + +def self_test_column_rejections(base, context): + reject = lambda mutator: reject_self_test_mutation(base, context, mutator) + reject(lambda records: records[2].__setitem__("num_values", None)) + reject(lambda records: records[2].__setitem__("num_values", "0001")) + reject(lambda records: records[2].__setitem__("num_values", str(1 << 63))) + reject(lambda records: records[2].__setitem__("nan_count", "0")) + reject(lambda records: records[2].__setitem__("min_value_hex", "00")) + reject(lambda records: records[2]["column_order"].__setitem__("field_id", 2)) + reject(lambda records: records[2]["column_order"].__setitem__( + "header_hex", "1c00")) + reject(lambda records: records[2].__setitem__( + "unknown_statistics_field_ids", [3, 3])) + reject(lambda records: records[2].__setitem__( + "unknown_statistics_field_ids", [9])) + reject(lambda records: records[1].__setitem__("file", "/absolute.parquet")) + reject(lambda records: records[1].__setitem__("file", "data/./binary.parquet")) + reject(lambda records: records[1].__setitem__( + "footer_length", records[1]["size"])) + reject(lambda records: records[1].__setitem__("footer_length", 0)) + reject(lambda records: records[1].__setitem__("column_order_count", 2)) + reject(lambda records: records[1].__setitem__("column_order_count", None)) + reject(lambda records: records[2]["column_order"].update({ + "state": "ABSENT", "field_id": None, "wire_type": None, + "header_hex": None})) + reject(lambda records: records[2]["leaf_schema"].update({ + "physical_type": "INT32", "logical_type": "FLOAT16"})) + return None + + +def self_test_leaf_rejections(base, context): + reject = lambda fields: reject_self_test_leaf(base, context, fields) + reject({"physical_type": "FIXED_LEN_BYTE_ARRAY", "type_length": None}) + reject({"physical_type": "FIXED_LEN_BYTE_ARRAY", "type_length": 0}) + reject({"logical_type": "STRING", "converted_type": None}) + reject({"physical_type": "INT32", "logical_type": "STRING", + "converted_type": "UTF8"}) + reject({"logical_type": "ENUM", "converted_type": "UTF8"}) + reject({"physical_type": "INT32", "logical_type": "DECIMAL", + "converted_type": "DECIMAL", "precision": 10, "scale": 2}) + reject({"physical_type": "BYTE_ARRAY", "logical_type": "DECIMAL", + "converted_type": "DECIMAL", "precision": 2, "scale": 3}) + reject({"precision": 2, "scale": 0}) + reject({"physical_type": "INT32", "logical_type": "INTEGER", + "converted_type": "UINT_16", "bit_width": 32, "is_signed": False}) + reject({"physical_type": "INT32", "logical_type": "INTEGER", + "converted_type": "UINT_64", "bit_width": 64, "is_signed": False}) + reject({"physical_type": "INT64", "logical_type": "TIME", + "converted_type": "TIME_MILLIS", "time_unit": "MICROS", + "is_adjusted_to_utc": True}) + reject({"physical_type": "BYTE_ARRAY", "logical_type": "GEOMETRY", + "geography_algorithm": "SPHERICAL"}) + reject({"crs": "OGC:CRS84"}) + reject({"physical_type": "FIXED_LEN_BYTE_ARRAY", "logical_type": "UUID", + "type_length": 15}) + reject({"physical_type": "FIXED_LEN_BYTE_ARRAY", "logical_type": "INTERVAL", + "type_length": 12}) + reject({"logical_type": "MAP", "converted_type": "MAP"}) + return None + + +def self_test_record_rejections(base, context, capabilities): + zero = "0" * 64 + reject = lambda mutator: reject_self_test_mutation(base, context, mutator) + reject(lambda records: records.insert(0, records.pop(1))) + reject(lambda records: records[3].__setitem__("status", "UNSUPPORTED")) + reject(lambda records: records[0].__setitem__("producer_version", "wrong")) + reject(lambda records: records[0].__setitem__("source_revision", "wrong")) + reject(lambda records: records[0].__setitem__("toolchain_sha256", zero)) + reject(lambda records: records[0].__setitem__("manifest_sha256", zero)) + reject(lambda records: records[0].__setitem__("artifact_manifest_sha256", zero)) + reject(lambda records: records[3].__setitem__("capability_id", "read.logical-values")) + no_file = copy.deepcopy(base) + pyarrow = next(item for item in capabilities["authority"] + if item["id"] == "pyarrow") + no_file[0].update({ + "producer": pyarrow["id"], + "producer_version": pyarrow["version"], + "source_revision": pyarrow["revision"], + "toolchain_sha256": pyarrow["toolchain_sha256"][0], + }) + no_file[3]["capability_id"] = "read.logical-values" + reject_self_test_records([no_file[0], no_file[3]], context) + planned_capabilities = copy.deepcopy(capabilities) + raw = next(item for item in planned_capabilities["authority"] + if item["id"] == "n6-raw-java") + claim = next(item for item in raw["claim"] + if item["capability"] == "wire.column-order.type" and + "apache-binary" in item["cases"]) + claim["status"] = "planned" + planned_context = (context[0], context[1], planned_capabilities, + context[3], context[4]) + reject_self_test_records(base, planned_context, gate=True) + reject_self_test_records([[]] + copy.deepcopy(base[1:]), context) + malformed_later = copy.deepcopy(base) + malformed_later[2] = [] + reject_self_test_records(malformed_later, context) + return None + + +def self_test_semantic_case(base, context, capabilities, fixtures): + authority = next(item for item in capabilities["authority"] + if item["id"] == "n6-independent-model") + case = next(item for item in fixtures["semantic_case"] + if item["id"] == "plain-bound-decoding") + capability = "semantic.type-order" + digest = case["expected_sha256"][capability] + run = copy.deepcopy(base[0]) + run.update({ + "evidence_id": "semantic-self-test", + "producer": authority["id"], + "producer_version": authority["version"], + "source_revision": authority["revision"], + "toolchain_sha256": authority["toolchain_sha256"][0], + }) + result = { + "record": "case_result", + "schema_version": 2, + "case_id": case["id"], + "capability_id": capability, + "digest_contract": case["digest_contract"], + "status": "PASS", + "expected_sha256": digest, + "actual_sha256": digest, + "detail": "semantic self-test", + } + records = [run, result] + validate_self_test_records(records, context) + validate_self_test_records(records, context, gate=True) + wrong = copy.deepcopy(records) + wrong[1]["expected_sha256"] = "1" * 64 + wrong[1]["actual_sha256"] = "1" * 64 + reject_self_test_records(wrong, context) + with_file = copy.deepcopy(records) + file_record = copy.deepcopy(base[1]) + file_record["case_id"] = case["id"] + with_file.insert(1, file_record) + reject_self_test_records(with_file, context) + return None + + +def self_test_gate_bindings(manifest_path, manifest, capabilities, fixtures): + normalized = [entry for section in ("frozen_evidence", "planned_evidence") + for entry in manifest[section] if entry["format"] == "normalized-jsonl"] + if not normalized: + raise EvidenceError("normalized evidence is absent from the manifest") + seed = copy.deepcopy(normalized[0]) + synthetic = copy.deepcopy(manifest) + synthetic["frozen_evidence"] = [entry + for entry in synthetic["frozen_evidence"] + if entry["format"] != "normalized-jsonl"] + synthetic["planned_evidence"] = [] + planned = copy.deepcopy(seed) + planned["status"] = "planned" + synthetic["planned_evidence"].append(planned) + try: + checked_gate_input(planned["file"], synthetic, manifest_path, set(), + evidence_limits(synthetic, fixtures, capabilities)) + except EvidenceError: + pass + else: + raise EvidenceError("planned evidence passed the gate binding self-test") + if safe_relative("evidence/\nunsafe.jsonl") or safe_relative("évidence.jsonl"): + raise EvidenceError("unsafe relative evidence path passed self-test") + if safe_evidence_file("test/conformance/n6/README.md") or \ + safe_evidence_file("test/conformance/n6/evidence/.jsonl"): + raise EvidenceError("unsafe evidence exclusion path passed self-test") + synthetic["planned_evidence"] = [] + entry = copy.deepcopy(seed) + entry.update({ + "status": "verified", + "storage": "checked-in", + "schema_sha256": synthetic["evidence_schema_sha256"], + }) + synthetic["frozen_evidence"].append(entry) + limits = evidence_limits(synthetic, fixtures, capabilities) + used_entries = set() + checked_gate_input(entry["file"], synthetic, manifest_path, used_entries, + limits) + try: + checked_gate_input(entry["file"], synthetic, manifest_path, + used_entries, limits) + except EvidenceError: + pass + else: + raise EvidenceError("duplicate frozen evidence passed its self-test") + validate_gate_entry_coverage(synthetic, used_entries) + try: + validate_gate_entry_coverage(synthetic, set()) + except EvidenceError: + pass + else: + raise EvidenceError("omitted frozen evidence passed its self-test") + return None + + +def self_test_jsonl(manifest, capabilities, fixtures): + limits = evidence_limits(manifest, fixtures, capabilities) + with tempfile.TemporaryDirectory() as directory: + temporary = pathlib.Path(directory) + + def reject_jsonl(name, payload, changed_limits=None): + path = temporary / name + path.write_bytes(payload) + try: + read_jsonl(path, changed_limits or limits) + except (EvidenceError, json.JSONDecodeError, UnicodeError): + return None + raise EvidenceError("an adversarial JSONL self-test passed") + + valid = temporary / "valid.jsonl" + valid.write_bytes(b"{}\n") + if read_jsonl(valid, limits) != [{}]: + raise EvidenceError("valid JSONL self-test failed") + linked = temporary / "linked.jsonl" + linked.symlink_to(valid) + try: + read_jsonl(linked, limits) + except EvidenceError: + pass + else: + raise EvidenceError("symbolic-link evidence passed its self-test") + empty = temporary / "empty.jsonl" + empty.write_bytes(b"") + try: + check_evidence_file_metadata(empty, "empty.jsonl", limits) + except EvidenceError: + pass + else: + raise EvidenceError("empty evidence passed its metadata self-test") + tiny_file = dict(limits, max_file_bytes=2) + try: + check_evidence_file_metadata(valid, "valid.jsonl", tiny_file) + except EvidenceError: + pass + else: + raise EvidenceError("oversized evidence passed its metadata self-test") + reject_jsonl("duplicate.jsonl", b'{"a":1,"a":2}\n') + reject_jsonl("constant.jsonl", b'{"a":NaN}\n') + reject_jsonl("float.jsonl", b'{"a":0.0}\n') + reject_jsonl("negative-zero.jsonl", b'{"a":-0}\n') + reject_jsonl("newline.jsonl", b"{}") + short_line = dict(limits, max_line_bytes=2) + reject_jsonl("line.jsonl", b"{} \n", short_line) + one_record = dict(limits, max_records_per_input=1) + reject_jsonl("records.jsonl", b"{}\n{}\n", one_record) + return None + + +def self_test_upstream_evidence(): + raw_entry = {"id": "raw", "file": + "test/conformance/n6/evidence/raw.jsonl"} + derived_entry = {"id": "derived", "file": + "test/conformance/n6/evidence/derived.jsonl", + "upstream_evidence": ["raw"]} + raw = { + "entry": raw_entry, + "run": {"evidence_id": "raw"}, + "sha256": "1" * 64, + } + derived = { + "entry": derived_entry, + "run": { + "evidence_id": "derived", + "upstream_evidence": [{ + "evidence_id": "raw", + "file": raw_entry["file"], + "sha256": raw["sha256"], + }], + }, + "sha256": "2" * 64, + } + inputs = {"raw": raw, "derived": derived} + validate_upstream_evidence(inputs) + for field, value in (("file", "test/conformance/n6/evidence/wrong.jsonl"), + ("sha256", "3" * 64), ("evidence_id", "absent")): + mutated = copy.deepcopy(inputs) + mutated["derived"]["run"]["upstream_evidence"][0][field] = value + try: + validate_upstream_evidence(mutated) + except EvidenceError: + pass + else: + raise EvidenceError( + f"invalid upstream {field} passed its self-test") + cyclic = copy.deepcopy(inputs) + cyclic["raw"]["entry"]["upstream_evidence"] = ["derived"] + cyclic["raw"]["run"]["upstream_evidence"] = [{ + "evidence_id": "derived", + "file": derived_entry["file"], + "sha256": derived["sha256"], + }] + try: + validate_upstream_evidence(cyclic) + except EvidenceError: + pass + else: + raise EvidenceError("cyclic upstream evidence passed its self-test") + return None + + +def run_self_test(validator, manifest_path, manifest, capabilities, fixtures, + input_hashes): + context = (validator, manifest, capabilities, fixtures, input_hashes) + base = base_self_test(manifest, capabilities, fixtures, input_hashes) + self_test_compact_headers() + validate_self_test_records(base, context) + self_test_valid_leaves(base, context) + self_test_multileaf_coverage(base, context, fixtures) + self_test_multirow_coverage(base, context, fixtures) + self_test_column_rejections(base, context) + self_test_leaf_rejections(base, context) + self_test_record_rejections(base, context, capabilities) + self_test_semantic_case(base, context, capabilities, fixtures) + self_test_gate_bindings(manifest_path, manifest, capabilities, fixtures) + self_test_jsonl(manifest, capabilities, fixtures) + self_test_upstream_evidence() + return None + + +def parse_arguments(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--schema", required=True) + parser.add_argument("--manifest", required=True) + parser.add_argument("--capabilities", required=True) + parser.add_argument("--fixtures", required=True) + parser.add_argument("--self-test", action="store_true") + parser.add_argument("--gate", action="store_true") + parser.add_argument("evidence", nargs="*") + return parser.parse_args(argv) + + +def load_validation_inputs(args): + manifest_bytes = stable_regular_bytes(args.manifest, CONTROL_FILE_LIMIT, + "N6 manifest", allow_empty=False) + manifest = tomllib.loads(manifest_bytes.decode("utf-8")) + root = repository_root(args.manifest) + schema_bytes = stable_regular_bytes(args.schema, CONTROL_FILE_LIMIT, + "N6 evidence schema", allow_empty=False) + capabilities_bytes = stable_regular_bytes(args.capabilities, + CONTROL_FILE_LIMIT, "N6 capabilities", allow_empty=False) + fixtures_bytes = stable_regular_bytes(args.fixtures, CONTROL_FILE_LIMIT, + "N6 fixtures", allow_empty=False) + + def unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise EvidenceError(f"N6 evidence schema has duplicate key {key}") + value[key] = item + return value + + schema = json.loads(schema_bytes, object_pairs_hook=unique_object) + validator_class = jsonschema.validators.validator_for(schema) + validator_class.check_schema(schema) + validator = validator_class(schema) + capabilities = tomllib.loads(capabilities_bytes.decode("utf-8")) + fixtures = tomllib.loads(fixtures_bytes.decode("utf-8")) + validate_model_producer_binding(root, manifest, capabilities) + fixtures["semantic_case"] = load_semantic_cases(root, manifest, + capabilities, fixtures) + plan_bytes = stable_regular_bytes(manifest_input_file(root, + manifest["plan_file"]), CONTROL_FILE_LIMIT, "N6 plan", allow_empty=False) + corpus_bytes = stable_regular_bytes(manifest_input_file(root, + manifest["corpus_manifest_file"]), CONTROL_FILE_LIMIT, + "N6 corpus manifest", allow_empty=False) + artifacts_bytes = stable_regular_bytes(manifest_input_file(root, + manifest["artifact_manifest_file"]), CONTROL_FILE_LIMIT, + "N6 artifact manifest", allow_empty=False) + input_hashes = { + "run": { + "plan_sha256": sha256_bytes(plan_bytes), + "capabilities_sha256": sha256_bytes(capabilities_bytes), + "fixture_manifest_sha256": sha256_bytes(fixtures_bytes), + "corpus_manifest_sha256": sha256_bytes(corpus_bytes), + "evidence_schema_sha256": sha256_bytes(schema_bytes), + }, + "artifact_manifest_sha256": sha256_bytes(artifacts_bytes), + "manifest_sha256": sha256_bytes(manifest_bytes), + } + limits = evidence_limits(manifest, fixtures, capabilities) + return validator, manifest, capabilities, fixtures, limits, input_hashes + + +def resolve_evidence_input(argument, args, manifest, used_entries, limits): + if args.gate: + return checked_gate_input(argument, manifest, args.manifest, + used_entries, limits) + return None, pathlib.Path(argument) + + +def validate_frozen_run(entry, run, records, validated): + if entry is None: + return None + if run["evidence_id"] != entry["id"]: + raise EvidenceError("run evidence ID does not match its frozen entry") + if run["producer"] != entry["authority"]: + raise EvidenceError("run producer does not match its frozen entry") + if run["toolchain_sha256"] != entry["toolchain_sha256"]: + raise EvidenceError("run toolchain does not match its frozen entry") + if len(records) != entry["record_count"]: + raise EvidenceError("record count does not match its frozen entry") + if len(validated["files"]) != entry["case_count"]: + raise EvidenceError("case count does not match its frozen entry") + return None + + +def collect_evidence_facts(validated, entry, records, evidence_ids, producers, + passing_results, file_facts, column_facts, column_cases): + run = validated["run"] + if run["evidence_id"] in evidence_ids: + raise EvidenceError(f"duplicate run evidence ID: {run['evidence_id']}") + evidence_ids.add(run["evidence_id"]) + coverage = producers.setdefault(run["producer"], { + "files": set(), "columns": set(), "results": set()}) + current = { + "files": set(validated["files"]), + "columns": set(validated["columns"]), + "results": set(validated["results"]), + } + for kind, keys in current.items(): + overlap = coverage[kind] & keys + if overlap: + raise EvidenceError( + f"producer evidence coverage overlaps for {kind}: {sorted(overlap)}") + coverage[kind].update(keys) + validate_frozen_run(entry, run, records, validated) + for case_id, record in validated["files"].items(): + file_facts.setdefault(case_id, set()).add(canonical_record(record)) + for key, record in validated["columns"].items(): + column_facts.setdefault(key, set()).add(canonical_record(record)) + column_cases.update(validated["column_cases"]) + for key in validated["passes"]: + passing_results.setdefault(key, []).append( + (run["producer"], validated["results"][key])) + return None + + +def validate_upstream_evidence(evidence_inputs): + graph = {} + for evidence_id, current in evidence_inputs.items(): + expected = current["entry"].get("upstream_evidence", []) + bindings = current["run"].get("upstream_evidence", []) + actual = [binding["evidence_id"] for binding in bindings] + if len(actual) != len(set(actual)): + raise EvidenceError(f"duplicate upstream evidence binding: {evidence_id}") + if sorted(actual) != sorted(expected): + raise EvidenceError( + f"upstream evidence set differs from its manifest: {evidence_id}") + graph[evidence_id] = set(actual) + for binding in bindings: + upstream_id = binding["evidence_id"] + if upstream_id not in evidence_inputs: + raise EvidenceError( + f"upstream evidence input is absent: {upstream_id}") + upstream = evidence_inputs[upstream_id] + if binding["file"] != upstream["entry"]["file"]: + raise EvidenceError( + f"upstream evidence path differs: {upstream_id}") + if binding["sha256"] != upstream["sha256"]: + raise EvidenceError( + f"upstream evidence digest differs: {upstream_id}") + visiting = set() + visited = set() + + def visit(evidence_id): + if evidence_id in visiting: + raise EvidenceError("upstream evidence graph has a cycle") + if evidence_id in visited: + return + visiting.add(evidence_id) + for upstream_id in graph[evidence_id]: + visit(upstream_id) + visiting.remove(evidence_id) + visited.add(evidence_id) + + for evidence_id in graph: + visit(evidence_id) + return None + + +def validate_authority_coverage(producers, capabilities): + authorities = authority_map(capabilities) + for producer, coverage in producers.items(): + claims = claim_statuses(authorities[producer]) + expected = {key for key, status in claims.items() + if status != "not_assessed"} + if coverage["results"] != expected: + missing = sorted(expected - coverage["results"]) + extra = sorted(coverage["results"] - expected) + raise EvidenceError( + f"producer claim coverage differs for {producer}: " + f"missing={missing}, extra={extra}") + return None + + +def validate_evidence_inputs(args, validator, manifest, capabilities, fixtures, + limits, input_hashes): + if len(args.evidence) > limits["max_inputs"]: + raise EvidenceError("too many evidence inputs") + used_entries = set() + evidence_ids = set() + producers = {} + evidence_inputs = {} + passing_results = {} + file_facts = {} + column_facts = {} + column_cases = set() + total_bytes = 0 + total_records = 0 + for evidence_argument in args.evidence: + entry, evidence_path = resolve_evidence_input(evidence_argument, args, + manifest, used_entries, limits) + evidence_bytes = stable_regular_bytes(evidence_path, + limits["max_file_bytes"], f"evidence input {evidence_argument}", + allow_empty=False) + evidence_sha256 = sha256_bytes(evidence_bytes) + if entry is not None and evidence_sha256 != entry["sha256"]: + raise EvidenceError( + f"gate evidence hash does not match its manifest: {evidence_argument}") + total_bytes += len(evidence_bytes) + if total_bytes > limits["max_total_bytes"]: + raise EvidenceError("evidence inputs exceed their total byte limit") + records = read_jsonl_bytes(evidence_bytes, evidence_argument, limits) + total_records += len(records) + if total_records > limits["max_records_total"]: + raise EvidenceError("evidence inputs exceed their total record limit") + validated = validate_records(records, validator, manifest, capabilities, + fixtures, input_hashes, args.gate) + run = validated["run"] + declared_entry = entry or manifest_evidence_entry(manifest, + run["evidence_id"]) + if run["evidence_id"] != declared_entry["id"] or \ + run["producer"] != declared_entry["authority"]: + raise EvidenceError("run identity differs from its manifest entry") + evidence_inputs[run["evidence_id"]] = { + "entry": declared_entry, + "run": run, + "sha256": evidence_sha256, + } + collect_evidence_facts(validated, entry, records, evidence_ids, + producers, passing_results, file_facts, column_facts, column_cases) + if args.gate: + validate_gate_entry_coverage(manifest, used_entries) + validate_upstream_evidence(evidence_inputs) + if args.gate: + validate_authority_coverage(producers, capabilities) + return passing_results, file_facts, column_facts, column_cases + + +def validate_cross_producer_facts(passing_results, file_facts, column_facts, + column_cases, fixtures): + semantic_ids = {case["id"] for case in fixtures["semantic_case"]} + for case_id, facts in file_facts.items(): + if len(facts) != 1: + raise EvidenceError(f"producers disagree on file facts: {case_id}") + for key, facts in column_facts.items(): + if len(facts) != 1: + raise EvidenceError(f"producers disagree on column facts: {key}") + for key, producer_records in passing_results.items(): + observations = {(record["digest_contract"], record["actual_sha256"]) + for _, record in producer_records} + if len(observations) != 1: + raise EvidenceError(f"passing producers disagree: {key}") + if key[1] in COLUMN_EVIDENCE_CAPABILITIES and \ + key[0] not in column_cases and key[0] not in semantic_ids: + raise EvidenceError(f"column capability has no normalized facts: {key}") + return None + + +def required_gate_results(fixtures, capabilities): + declared = { + (case["id"], capability) + for case in fixtures["fixture"] + fixtures["generated_case"] + for capability in case["capabilities"] + } + statuses = {} + for authority in capabilities["authority"]: + for claim in authority.get("claim", []): + for case_id in claim["cases"]: + statuses.setdefault((case_id, claim["capability"]), set()).add( + claim["status"]) + unreviewed = sorted(declared - set(statuses)) + if unreviewed: + raise EvidenceError(f"fixture capabilities lack reviewed claims: {unreviewed}") + positive = {key for key, values in statuses.items() + if values & {"verified", "planned"}} + return declared & positive + + +def self_test_required_gate_results(): + fixtures = {"fixture": [{"id": "case", "capabilities": ["cap"]}], + "generated_case": []} + capabilities = {"authority": [{"claim": [{"capability": "cap", + "status": "unsupported", "cases": ["case"]}]}]} + if required_gate_results(fixtures, capabilities): + raise EvidenceError("unsupported-only claim became a required pass") + capabilities["authority"][0]["claim"][0]["status"] = "verified" + if required_gate_results(fixtures, capabilities) != {("case", "cap")}: + raise EvidenceError("verified claim did not become a required pass") + capabilities["authority"][0]["claim"] = [] + try: + required_gate_results(fixtures, capabilities) + except EvidenceError: + pass + else: + raise EvidenceError("unreviewed fixture capability passed its self-test") + return None + + +def validate_gate_results(fixtures, capabilities, passing_results): + required = required_gate_results(fixtures, capabilities) + missing = sorted(required - set(passing_results)) + if missing: + raise EvidenceError(f"missing passing capability evidence: {missing}") + validate_comparison_groups(fixtures, passing_results) + return None + + +def main(argv=None): + args = parse_arguments(argv) + validator, manifest, capabilities, fixtures, limits, input_hashes = \ + load_validation_inputs(args) + if args.gate: + validate_gate_bindings(args, manifest, input_hashes) + if args.self_test: + run_self_test(validator, args.manifest, manifest, capabilities, fixtures, + input_hashes) + self_test_required_gate_results() + passing, files, columns, column_cases = validate_evidence_inputs(args, + validator, manifest, capabilities, fixtures, limits, input_hashes) + validate_cross_producer_facts(passing, files, columns, column_cases, + fixtures) + if args.gate: + validate_gate_results(fixtures, capabilities, passing) + print("N6 normalized evidence validation passed.") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (ValueError, OSError, UnicodeError, jsonschema.ValidationError) as error: + print(f"N6 evidence validation failed: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/test/delta.jl b/test/delta.jl new file mode 100644 index 0000000..54574f0 --- /dev/null +++ b/test/delta.jl @@ -0,0 +1,452 @@ +using Random + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end + +const DELTA_CORPUS = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + +function deltacorpus(parts...) + return joinpath(DELTA_CORPUS, "data", parts...) +end + +function deltauleb!(output::Vector{UInt8}, value::Integer) + value = UInt64(value) + while value >= 0x80 + push!(output, UInt8(value & 0x7f) | 0x80) + value >>= 7 + end + push!(output, UInt8(value)) + return +end + +function deltazigzag!(output::Vector{UInt8}, value::Int64) + deltauleb!(output, reinterpret(UInt64, (value << 1) ⊻ (value >> 63))) + return +end + +function deltazigzag!(output::Vector{UInt8}, value::Int32) + deltauleb!(output, reinterpret(UInt32, (value << 1) ⊻ (value >> 31))) + return +end + +function deltapackbits!(output::Vector{UInt8}, values, width::Int, slots::Int) + width == 0 && return + bits = falses(slots * width) + for (index, value) in enumerate(values), bit in 0:(width - 1) + bits[(index - 1) * width + bit + 1] = (UInt64(value) >> bit) & 0x01 == 0x01 + end + for byte in 1:(length(bits) ÷ 8) + packed = UInt8(0) + for bit in 0:7 + bits[(byte - 1) * 8 + bit + 1] && (packed |= UInt8(1) << bit) + end + push!(output, packed) + end + return +end + +# Independent reference encoder with a configurable block layout (bit-by-bit packing). +function referencedelta(::Type{T}, values; blocksize::Int=128, miniblocks::Int=4) where {T} + output = UInt8[] + deltauleb!(output, blocksize) + deltauleb!(output, miniblocks) + deltauleb!(output, length(values)) + deltazigzag!(output, isempty(values) ? zero(T) : T(values[1])) + slots = blocksize ÷ miniblocks + deltas = T[T(values[i]) - T(values[i - 1]) for i in 2:length(values)] + for block in Iterators.partition(deltas, blocksize) + low = minimum(block) + deltazigzag!(output, low) + relative = [UInt64(reinterpret(unsigned(T), delta - low)) for delta in block] + chunks = [relative[((m - 1) * slots + 1):min(m * slots, end)] for m in 1:miniblocks if (m - 1) * slots < length(relative)] + widths = zeros(UInt8, miniblocks) + for (m, chunk) in enumerate(chunks) + widths[m] = UInt8(64 - leading_zeros(maximum(chunk))) + end + append!(output, widths) + for (m, chunk) in enumerate(chunks) + deltapackbits!(output, chunk, Int(widths[m]), slots) + end + end + return output +end + +function deltaheaderbytes(blocksize::Integer, miniblocks::Integer, count::Integer, first::Int64) + output = UInt8[] + deltauleb!(output, blocksize) + deltauleb!(output, miniblocks) + deltauleb!(output, count) + deltazigzag!(output, first) + return output +end + +function fixturecsvline(line::String) + fields = String[] + index = firstindex(line) + while true + if index <= lastindex(line) && line[index] == '"' + close = findnext('"', line, index + 1) + push!(fields, line[(index + 1):(close - 1)]) + index = close + 1 + index > lastindex(line) && break + line[index] == ',' || error("unexpected CSV character") + index += 1 + index > lastindex(line) && (push!(fields, ""); break) + else + comma = findnext(',', line, index) + comma === nothing && (push!(fields, line[index:end]); break) + push!(fields, line[index:(comma - 1)]) + index = comma + 1 + index > lastindex(line) && (push!(fields, ""); break) + end + end + return fields +end + +function fixturecsv(path::String) + lines = readlines(path) + return fixturecsvline(lines[1]), [fixturecsvline(line) for line in lines[2:end]] +end + +function fixturefooter(path::String) + file = Parquet.File(path) + meta = TH.decode(copy(file.footer.bytes), MD.FileMetaData) + close(file) + return meta +end + +# The data section of the single V2 data page of a column chunk (levels skipped by length). +function fixturepage(path::String, column::Int) + meta = fixturefooter(path) + bytes = read(path) + chunk = meta.row_groups[1].columns[column] + md = chunk.meta_data + r = TH.Reader(bytes, md.data_page_offset + 1, length(bytes)) + header = TH.decode(r, MD.PageHeader) + v2 = header.data_page_header_v2 + start = md.data_page_offset + TH.consumed(r) + v2.definition_levels_byte_length + v2.repetition_levels_byte_length + 1 + stop = md.data_page_offset + TH.consumed(r) + header.compressed_page_size + return view(bytes, start:stop), v2.num_values - v2.num_nulls, md.type_, meta.schema[column + 1].name, md, header +end + +@testset "DELTA_BINARY_PACKED specification examples" begin + @test Parquet.encode_delta_binary_packed(Int32[1, 2, 3, 4, 5]) == UInt8[0x80, 0x01, 0x04, 0x05, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00] + example = UInt8[0x80, 0x01, 0x04, 0x08, 0x0e, 0x03, 0x02, 0x00, 0x00, 0x00, 0xc0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + values = Int32[7, 5, 3, 1, 2, 3, 4, 5] + @test Parquet.encode_delta_binary_packed(values) == example + @test referencedelta(Int32, values) == example + @test Parquet.decode_delta_binary_packed(Int32, example, 8) == (values, 19) + @test Parquet.decode_delta_binary_packed(Int64, example, 8)[1] == Int64.(values) + @test Parquet.decode_delta_binary_packed(Int32, vcat(example, UInt8[0xff]), 8)[2] == 19 + @test Parquet.encode_delta_binary_packed(Int32[]) == UInt8[0x80, 0x01, 0x04, 0x00, 0x00] + @test Parquet.decode_delta_binary_packed(Int32, UInt8[0x80, 0x01, 0x04, 0x00, 0x00], 0) == (Int32[], 6) + @test Parquet.encode_delta_binary_packed(Int64[-1]) == UInt8[0x80, 0x01, 0x04, 0x01, 0x01] + @test Parquet.decode_delta_binary_packed(Int64, UInt8[0x80, 0x01, 0x04, 0x01, 0x01], 1) == (Int64[-1], 6) + output = zeros(Int32, 8) + padded = vcat(UInt8[0xaa, 0xbb], example) + @test Parquet.decode_delta_binary_packed!(output, padded; offset=3) == length(padded) + 1 + @test output == values + @test Parquet.decode_delta_binary_packed(Int32, view(padded, 3:length(padded)), 8)[1] == values + slice = Parquet.readrange(Parquet.source(padded), 2, length(padded) - 2) + @test Parquet.decode_delta_binary_packed(Int32, slice, 8) == (values, length(example) + 1) +end + +@testset "DELTA_BINARY_PACKED layouts and boundaries" begin + rng = MersenneTwister(20260821) + for T in (Int32, Int64), count in (0, 1, 2, 31, 32, 33, 127, 128, 129, 130, 255, 256, 257, 1000) + values = rand(rng, T, count) + encoded = Parquet.encode_delta_binary_packed(values) + @test encoded == referencedelta(T, values) + @test Parquet.decode_delta_binary_packed(T, encoded, count) == (values, length(encoded) + 1) + for (blocksize, miniblocks) in ((128, 1), (128, 2), (256, 8), (512, 4), (1024, 32)) + stream = referencedelta(T, values; blocksize=blocksize, miniblocks=miniblocks) + @test Parquet.decode_delta_binary_packed(T, stream, count) == (values, length(stream) + 1) + end + end + for width in 0:63 + limit = width == 63 ? typemax(Int64) : (Int64(1) << width) - 1 + values = rand(rng, Int64(0):limit, 300) + encoded = Parquet.encode_delta_binary_packed(values) + @test encoded == referencedelta(Int64, values) + @test Parquet.decode_delta_binary_packed(Int64, encoded, 300)[1] == values + end + for values in (fill(Int32(-7), 200), Int32.(-100:99), Int32[typemax(Int32), typemin(Int32), typemax(Int32)], + Int32[typemin(Int32), typemax(Int32)], Int64[typemax(Int64), typemin(Int64), 0, -1, typemax(Int64)], + Int64[typemin(Int64)], Int64.(-(1:129)), Int32[0, typemin(Int32)]) + encoded = Parquet.encode_delta_binary_packed(values) + @test encoded == referencedelta(eltype(values), values) + @test Parquet.decode_delta_binary_packed(eltype(values), encoded, length(values))[1] == values + end +end + +@testset "DELTA_BINARY_PACKED two's-complement wrapping" begin + for values in (Int32[typemax(Int32), typemin(Int32)], Int32[typemin(Int32), typemax(Int32)], + Int32[typemax(Int32), typemin(Int32), typemax(Int32), typemin(Int32)], Int32[0, typemax(Int32), typemin(Int32), 0], + Int64[typemax(Int64), typemin(Int64)], Int64[typemin(Int64), typemax(Int64)], + Int64[typemax(Int64), typemin(Int64), typemax(Int64), typemin(Int64)], Int64[-1, typemax(Int64), typemin(Int64), 1], + Int32[typemin(Int32), 0, typemax(Int32), typemin(Int32) + 1], Int64[typemax(Int64) - 1, typemin(Int64) + 1]) + T = eltype(values) + encoded = Parquet.encode_delta_binary_packed(values) + @test encoded == referencedelta(T, values) + decoded, next = Parquet.decode_delta_binary_packed(T, encoded, length(values)) + @test decoded == values && next == length(encoded) + 1 + @test reinterpret(unsigned(T), decoded) == reinterpret(unsigned(T), values) + end + # typemax -> typemin wraps to a delta of +1 and typemin -> typemax to -1 at the physical width + for (values, mindelta) in ((Int32[typemax(Int32), typemin(Int32)], 0x02), (Int32[typemin(Int32), typemax(Int32)], 0x01), + (Int64[typemax(Int64), typemin(Int64)], 0x02), (Int64[typemin(Int64), typemax(Int64)], 0x01)) + headerlength = length(deltaheaderbytes(128, 4, 2, Int64(values[1]))) + encoded = Parquet.encode_delta_binary_packed(values) + @test encoded[headerlength + 1] == mindelta && encoded[(headerlength + 2):end] == UInt8[0x00, 0x00, 0x00, 0x00] + end + # INT32 streams must stay within the physical width: 33-bit miniblocks and out-of-range header values are rejected + wide = referencedelta(Int64, Int64[0, typemax(Int32), typemin(Int32)]) + @test Parquet.decode_delta_binary_packed(Int64, wide, 3)[1] == Int64[0, typemax(Int32), typemin(Int32)] + @test_throws Parquet.FormatError Parquet.decode_delta_binary_packed(Int32, wide, 3) + @test_throws Parquet.FormatError Parquet.decode_delta_binary_packed(Int32, referencedelta(Int64, Int64[Int64(2)^31]), 1) + @test_throws Parquet.FormatError Parquet.decode_delta_binary_packed(Int32, referencedelta(Int64, Int64[-Int64(2)^31 - 1]), 1) + @test_throws Parquet.FormatError Parquet.decode_delta_binary_packed(Int32, referencedelta(Int64, Int64[0, -Int64(2)^31 - 5]), 2) + @test_throws Parquet.FormatError Parquet.decode_delta_binary_packed(Int32, referencedelta(Int64, Int64[0, Int64(2)^31]), 2) + @test Parquet.decode_delta_binary_packed(Int32, referencedelta(Int64, Int64[0, typemin(Int32)]), 2)[1] == Int32[0, typemin(Int32)] + @test Parquet.decode_delta_binary_packed(Int32, referencedelta(Int32, Int32[0, typemin(Int32)]), 2)[1] == Int32[0, typemin(Int32)] +end + +@testset "DELTA_BINARY_PACKED unused miniblock widths and padding" begin + # values 1, 2, 3: one used miniblock of width 0; the three unused width bytes hold 0xff + unused = vcat(deltaheaderbytes(128, 4, 3, Int64(1)), UInt8[0x02, 0x00, 0xff, 0xff, 0xff]) + @test Parquet.decode_delta_binary_packed(Int64, unused, 3) == (Int64[1, 2, 3], length(unused) + 1) + @test Parquet.decode_delta_binary_packed(Int32, unused, 3) == (Int32[1, 2, 3], length(unused) + 1) + # a used miniblock still validates its width against the physical type + @test_throws Parquet.FormatError Parquet.decode_delta_binary_packed(Int64, vcat(deltaheaderbytes(128, 4, 3, Int64(1)), UInt8[0x02, 0xff, 0x00, 0x00, 0x00]), 3) + @test_throws Parquet.FormatError Parquet.decode_delta_binary_packed(Int32, vcat(deltaheaderbytes(128, 4, 3, Int64(1)), UInt8[0x02, 0x21, 0x00, 0x00, 0x00], zeros(UInt8, 132)), 3) + @test Parquet.decode_delta_binary_packed(Int64, vcat(deltaheaderbytes(128, 4, 3, Int64(1)), UInt8[0x02, 0x21, 0x00, 0x00, 0x00], zeros(UInt8, 132)), 3)[1] == Int64[1, 2, 3] + # padding bits of a partially used miniblock are arbitrary + padded = vcat(deltaheaderbytes(128, 4, 3, Int64(1)), UInt8[0x02, 0x01, 0xff, 0xff, 0xff, 0xfc, 0xff, 0xff, 0xff]) + @test Parquet.decode_delta_binary_packed(Int64, padded, 3) == (Int64[1, 2, 3], length(padded) + 1) + # the second block of a multi-block stream also tolerates unused width bytes + values = Int64.(1:140) + stream = referencedelta(Int64, values) + stream[end - 2:end] .= 0xff + @test Parquet.decode_delta_binary_packed(Int64, stream, 140)[1] == values +end + +@testset "DELTA_BINARY_PACKED malformed input and limits" begin + F = Parquet.FormatError + L = Parquet.LimitError + good = Parquet.encode_delta_binary_packed(Int64.(1:300)) + for n in 0:(length(good) - 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, good[1:n], 300) + end + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(0, 4, 1, Int64(0)), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(100, 4, 1, Int64(0)), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(Int64(2)^31, 4, 1, Int64(0)), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(128, 0, 1, Int64(0)), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(128, 3, 1, Int64(0)), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(128, 8, 1, Int64(0)), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(128, 129, 1, Int64(0)), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(128, 4, 5, Int64(0)), 8) + @test_throws F Parquet.decode_delta_binary_packed(Int64, deltaheaderbytes(128, 4, Int64(2)^32, Int64(0)), 8) + @test_throws F Parquet.decode_delta_binary_packed(Int64, vcat(deltaheaderbytes(128, 4, 3, Int64(0)), UInt8[0x00, 0x41, 0x00, 0x00, 0x00]), 3) + @test_throws F Parquet.decode_delta_binary_packed(Int64, vcat(deltaheaderbytes(128, 4, 3, Int64(0)), UInt8[0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00]), 3) + @test_throws F Parquet.decode_delta_binary_packed(Int64, vcat(deltaheaderbytes(128, 4, 3, Int64(0)), UInt8[0x00, 0x01, 0x00]), 3) + @test_throws F Parquet.decode_delta_binary_packed(Int64, fill(0x80, 11), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, vcat(fill(0x80, 9), UInt8[0x02]), 1) + @test_throws F Parquet.decode_delta_binary_packed(Int64, UInt8[0x80], 1) + @test_throws L Parquet.decode_delta_binary_packed(Int64, good, 300; limits=Parquet.Limits(max_container_elements=10)) + @test_throws L Parquet.decode_delta_binary_packed(Int64, good, 300; limits=Parquet.Limits(max_page_bytes=2000)) + # layout fields are charged to limits before any allocation + @test_throws L Parquet.decode_delta_binary_packed!(Vector{Int64}(undef, 2), deltaheaderbytes(128, 4, 2, Int64(0)); limits=Parquet.Limits(max_container_elements=100)) + huge = deltaheaderbytes(Int64(2)^30, Int64(2)^25, 2, Int64(0)) + @test_throws L Parquet.decode_delta_binary_packed!(Vector{Int64}(undef, 2), huge; limits=Parquet.Limits(max_container_elements=Int64(2)^20)) + @test_throws L Parquet.decode_delta_binary_packed!(Vector{Int64}(undef, 2), huge) + @test_throws F Parquet.decode_delta_binary_packed!(Vector{Int64}(undef, 2), huge; limits=Parquet.Limits(max_container_elements=Int64(2)^31)) + @test_throws L Parquet.decode_delta_binary_packed!(Vector{Int64}(undef, 2), vcat(deltaheaderbytes(Int64(2)^30, 1, 2, Int64(0)), UInt8[0x00, 0x40])) + @test_throws F Parquet.decode_delta_binary_packed!(Vector{Int64}(undef, 2), vcat(deltaheaderbytes(Int64(2)^30, 1, 2, Int64(0)), UInt8[0x00, 0x40]); limits=Parquet.Limits(max_page_bytes=Int64(2)^40, max_container_elements=Int64(2)^31)) + @test_throws F Parquet.decode_delta_binary_packed(Int32, vcat(deltaheaderbytes(128, 4, 3, Int64(0)), UInt8[0x00, 0x21, 0x00, 0x00, 0x00]), 3) + @test_throws ArgumentError Parquet.decode_delta_binary_packed(Int64, good, -1) + @test_throws L Parquet.decode_delta_binary_packed(Int64, good, big(typemax(Int64)) + 1) + @test Parquet.decode_delta_binary_packed!(Int64[], UInt8[0x80, 0x01, 0x04, 0x00, 0x00]) == 6 + rng = MersenneTwister(7) + for trial in 1:400 + mutated = copy(good) + for _ in 1:rand(rng, 1:3) + mutated[rand(rng, eachindex(mutated))] = rand(rng, UInt8) + end + result = try + Parquet.decode_delta_binary_packed(Int64, mutated, 300) + :ok + catch err + err + end + @test result === :ok || result isa Union{F,L} + end +end + +@testset "DELTA_LENGTH_BYTE_ARRAY" begin + F = Parquet.FormatError + L = Parquet.LimitError + words = ["Hello", "World", "Foobar", "ABCDEF"] + expected = [Vector{UInt8}(codeunits(word)) for word in words] + encoded = Parquet.encode_delta_length_byte_array(words) + @test encoded == vcat(Parquet.encode_delta_binary_packed(Int32[5, 5, 6, 6]), Vector{UInt8}(codeunits("HelloWorldFoobarABCDEF"))) + @test Parquet.encode_delta_length_byte_array(expected) == encoded + @test Parquet.decode_delta_length_byte_array(encoded, 4) == (expected, length(encoded) + 1) + offsets, next = Parquet.decode_delta_length_byte_array_offsets(encoded, 4) + @test length(offsets) == 5 && next == offsets[end] == length(encoded) + 1 + @test offsets[1] == length(Parquet.encode_delta_binary_packed(Int32[5, 5, 6, 6])) + 1 + @test String(encoded[offsets[3]:(offsets[4] - 1)]) == "Foobar" + mixed = [UInt8[], UInt8[0x00], UInt8[], UInt8[0xff, 0x00]] + @test Parquet.decode_delta_length_byte_array(Parquet.encode_delta_length_byte_array(mixed), 4)[1] == mixed + @test Parquet.decode_delta_length_byte_array(Parquet.encode_delta_length_byte_array(String[]), 0) == (Vector{UInt8}[], 6) + rng = MersenneTwister(3) + for count in (1, 7, 129, 1000) + values = [rand(rng, UInt8, rand(rng, 0:20)) for _ in 1:count] + stream = Parquet.encode_delta_length_byte_array(values) + padded = vcat(UInt8[0x01], stream) + @test Parquet.decode_delta_length_byte_array(padded, count; offset=2) == (values, length(padded) + 1) + slice = Parquet.readrange(Parquet.source(padded), 1, length(stream)) + @test Parquet.decode_delta_length_byte_array(slice, count)[1] == values + end + @test_throws F Parquet.decode_delta_length_byte_array(Parquet.encode_delta_binary_packed(Int32[-1]), 1) + @test_throws F Parquet.decode_delta_length_byte_array(Parquet.encode_delta_binary_packed(Int32[10]), 1) + @test_throws F Parquet.decode_delta_length_byte_array(encoded, 3) + @test_throws L Parquet.decode_delta_length_byte_array(encoded, 4; limits=Parquet.Limits(max_string_bytes=3)) + @test_throws L Parquet.decode_delta_length_byte_array(encoded, 4; limits=Parquet.Limits(max_page_bytes=10)) + @test_throws L Parquet.decode_delta_length_byte_array(encoded, 4; limits=Parquet.Limits(max_container_elements=2)) + for n in 0:(length(encoded) - 1) + @test_throws F Parquet.decode_delta_length_byte_array(encoded[1:n], 4) + end +end + +@testset "DELTA_BYTE_ARRAY" begin + F = Parquet.FormatError + L = Parquet.LimitError + words = ["axis", "axle", "babble", "babyhood"] + expected = [Vector{UInt8}(codeunits(word)) for word in words] + encoded = Parquet.encode_delta_byte_array(words) + @test encoded == vcat(Parquet.encode_delta_binary_packed(Int32[0, 2, 0, 3]), Parquet.encode_delta_length_byte_array(["axis", "le", "babble", "yhood"])) + @test Parquet.decode_delta_byte_array(encoded, 4) == (expected, length(encoded) + 1) + data, offsets, next = Parquet.decode_delta_byte_array_buffer(encoded, 4) + @test String(data) == "axisaxlebabblebabyhood" && offsets == [1, 5, 9, 15, 23] && next == length(encoded) + 1 + repeated = ["same", "same", "", "same", "sam", "samba"] + @test Parquet.decode_delta_binary_packed(Int32, Parquet.encode_delta_byte_array(repeated), 6)[1] == Int32[0, 4, 0, 0, 3, 3] + @test Parquet.decode_delta_byte_array(Parquet.encode_delta_byte_array(repeated), 6)[1] == [Vector{UInt8}(codeunits(w)) for w in repeated] + @test Parquet.decode_delta_byte_array(Parquet.encode_delta_byte_array(String[]), 0) == (Vector{UInt8}[], 11) + @test Parquet.decode_delta_byte_array(Parquet.encode_delta_byte_array([UInt8[0xff]]), 1)[1] == [UInt8[0xff]] + fixed = UInt8[1 1 2; 2 2 2; 3 4 4] + stream = Parquet.encode_delta_byte_array_fixed(fixed) + @test Parquet.decode_delta_byte_array_fixed(stream, 3, 3) == (fixed, length(stream) + 1) + @test_throws ArgumentError Parquet.encode_delta_byte_array_fixed(Matrix{UInt8}(undef, 0, 4)) + @test_throws F Parquet.decode_delta_byte_array_fixed(stream, 3, 0) + @test Parquet.decode_delta_byte_array_fixed(Parquet.encode_delta_byte_array_fixed(Matrix{UInt8}(undef, 2, 0)), 0, 2)[1] == Matrix{UInt8}(undef, 2, 0) + @test_throws F Parquet.decode_delta_byte_array_fixed(Parquet.encode_delta_byte_array(["ab", "abc"]), 2, 2) + @test_throws F Parquet.decode_delta_byte_array(vcat(Parquet.encode_delta_binary_packed(Int32[1]), Parquet.encode_delta_length_byte_array(["x"])), 1) + @test_throws F Parquet.decode_delta_byte_array(vcat(Parquet.encode_delta_binary_packed(Int32[0, 5]), Parquet.encode_delta_length_byte_array(["ab", "c"])), 2) + @test_throws F Parquet.decode_delta_byte_array(vcat(Parquet.encode_delta_binary_packed(Int32[-1]), Parquet.encode_delta_length_byte_array(["x"])), 1) + @test_throws F Parquet.decode_delta_byte_array(encoded, 3) + count = 200 + bomb = vcat(Parquet.encode_delta_binary_packed(Int32[i - 1 for i in 1:count]), Parquet.encode_delta_length_byte_array(fill("x", count))) + @test_throws L Parquet.decode_delta_byte_array(bomb, count; limits=Parquet.Limits(max_page_bytes=1000)) + @test_throws L Parquet.decode_delta_byte_array(bomb, count; limits=Parquet.Limits(max_string_bytes=100)) + materialized = Parquet.Limits(max_materialized_bytes=5000) + function rejectmaterializedbomb() + budget = Parquet._LiveByteBudget(materialized) + @test_throws L Parquet.decode_delta_byte_array(bomb, count; + limits=materialized, budget=budget) + @test Parquet._budgetused(budget) == 0 + return + end + rejectmaterializedbomb() + GC.gc() + @test @allocated(rejectmaterializedbomb()) < 10_000 + @test_throws L Parquet.decode_delta_byte_array_fixed(UInt8[], 0, big(typemax(Int64)) + 1) + @test length(Parquet.decode_delta_byte_array(bomb, count)[1][end]) == count + rng = MersenneTwister(11) + for count in (1, 5, 129, 700) + values = [rand(rng, UInt8[0x61, 0x62, 0x63], rand(rng, 0:12)) for _ in 1:count] + stream = Parquet.encode_delta_byte_array(values) + padded = vcat(UInt8[0x00, 0x00], stream) + @test Parquet.decode_delta_byte_array(padded, count; offset=3) == (values, length(padded) + 1) + slice = Parquet.readrange(Parquet.source(padded), 2, length(stream)) + @test Parquet.decode_delta_byte_array(slice, count)[1] == values + end + good = Parquet.encode_delta_byte_array(["alpha", "alphabet", "beta", "", "gamma"]) + for n in 0:(length(good) - 1) + @test_throws F Parquet.decode_delta_byte_array(good[1:n], 5) + end + for trial in 1:400 + mutated = copy(good) + for _ in 1:rand(rng, 1:3) + mutated[rand(rng, eachindex(mutated))] = rand(rng, UInt8) + end + result = try + Parquet.decode_delta_byte_array(mutated, 5) + :ok + catch err + err + end + @test result === :ok || result isa Union{F,L} + end +end + +@testset "delta encodings corpus fixtures" begin + if !isdir(deltacorpus()) + @warn "parquet-testing corpus not found; skipping delta fixture tests" DELTA_CORPUS + else + for (name, identical) in (("delta_binary_packed.parquet", false), ("delta_encoding_required_column.parquet", true), + ("delta_encoding_optional_column.parquet", true), ("delta_byte_array.parquet", false)) + path = deltacorpus(name) + header, rows = fixturecsv(deltacorpus(replace(name, ".parquet" => "_expect.csv"))) + for column in eachindex(header) + payload, count, type, _, _, _ = fixturepage(path, column) + expected = [row[column] for row in rows if row[column] != ""] + @test length(expected) == count + if type == MD.Type.INT32 || type == MD.Type.INT64 + ET = type == MD.Type.INT32 ? Int32 : Int64 + values, next = Parquet.decode_delta_binary_packed(ET, payload, count) + @test values == parse.(ET, expected) + @test next == length(payload) + 1 + reencoded = Parquet.encode_delta_binary_packed(values) + identical && @test reencoded == payload + @test Parquet.decode_delta_binary_packed(ET, reencoded, count)[1] == values + else + values, next = Parquet.decode_delta_byte_array(payload, count) + @test values == [Vector{UInt8}(codeunits(value)) for value in expected] + @test next == length(payload) + 1 + reencoded = Parquet.encode_delta_byte_array(values) + identical && @test reencoded == payload + @test Parquet.decode_delta_byte_array(reencoded, count)[1] == values + end + end + end + widths = fixturefooter(deltacorpus("delta_binary_packed.parquet")).schema[2:end] + @test [element.name for element in widths] == vcat(["bitwidth$i" for i in 0:64], ["int_value"]) + payload, count, _, name, _, _ = fixturepage(deltacorpus("delta_byte_array.parquet"), 7) + @test name == "c_login" && count == 0 + @test Parquet.decode_delta_byte_array(payload, 0) == (Vector{UInt8}[], length(payload) + 1) + end +end + +@testset "DELTA_LENGTH_BYTE_ARRAY zstd corpus fixture" begin + if !isdir(deltacorpus()) + @info "parquet-testing corpus is not available; skipping delta_length_byte_array.parquet" + else + path = deltacorpus("delta_length_byte_array.parquet") + compressed, count, type, name, md, header = fixturepage(path, 1) + v2 = header.data_page_header_v2 + payload = Parquet.decompress(md.codec, compressed, + header.uncompressed_page_size - v2.definition_levels_byte_length - + v2.repetition_levels_byte_length) + @test length(payload) == header.uncompressed_page_size - v2.definition_levels_byte_length + @test name == "FRUIT" && count == 1000 + values, next = Parquet.decode_delta_length_byte_array(payload, count) + @test next == length(payload) + 1 && length(values) == 1000 + @test md.statistics === nothing + @test values == [Vector{UInt8}(codeunits("apple_banana_mango$((index - 1)^2)")) for index in 1:1000] + @test Parquet.decode_delta_length_byte_array(Parquet.encode_delta_length_byte_array(values), count)[1] == values + end +end diff --git a/test/dictionary.jl b/test/dictionary.jl new file mode 100644 index 0000000..b636076 --- /dev/null +++ b/test/dictionary.jl @@ -0,0 +1,299 @@ +function dictionarypage(values; width=nothing, encoding=MD.Encoding.PLAIN, crc=:valid, + count=length(values), extra=UInt8[]) + payload = vcat(columnplain(values; width=width), extra) + header = MD.DictionaryPageHeader(num_values=Int32(count), encoding=encoding) + return columnpage(payload; type=MD.PageType.DICTIONARY_PAGE, dict=header, crc=crc) +end + +function dictionarydatapage(indices; levels=nothing, maxlevel=0, + bitwidth=Parquet._dictionarybitwidth(isempty(indices) ? 0 : Int(maximum(indices)) + 1), + encoding=MD.Encoding.RLE_DICTIONARY, extra=UInt8[], omit_indices=false, crc=:valid) + count = levels === nothing ? length(indices) : length(levels) + payload = levels === nothing ? UInt8[] : columnlevels(levels, maxlevel) + if !omit_indices + push!(payload, UInt8(bitwidth)) + append!(payload, Parquet.encode_hybrid(UInt64.(indices), bitwidth)) + end + append!(payload, extra) + return columnpage(payload; v1=columnv1(count; encoding=encoding), crc=crc) +end + +function readdictionarypages(dictionary, data, leaf; num_values, data_page_offset=nothing, + dictionary_page_offset=Int64(4), kwargs...) + dataoffset = something(data_page_offset, 4 + length(dictionary)) + pages = data isa Vector{UInt8} ? [dictionary, data] : vcat([dictionary], data) + return readsynthetic(pages, leaf; num_values=num_values, data_page_offset=dataoffset, + dictionary_page_offset=dictionary_page_offset, kwargs...) +end + +@testset "dictionary primitives" begin + @test Parquet._isdictionaryencoding(MD.Encoding.PLAIN_DICTIONARY) + @test Parquet._isdictionaryencoding(MD.Encoding.RLE_DICTIONARY) + @test !Parquet._isdictionaryencoding(MD.Encoding.PLAIN) + @test [Parquet._dictionarybitwidth(count) for count in 0:9] == [0, 0, 1, 2, 2, 3, 3, 3, 3, 4] + @test Parquet._encodedictionaryindices(zeros(UInt64, 128), 0) == UInt8[0x00, 0x80, 0x02] + @test Parquet._encodedictionaryindices(fill(UInt64(0x123), 3), 9) == UInt8[0x09, 0x06, 0x23, 0x01] + encoded = vcat(UInt8[0x02], Parquet.encode_hybrid(UInt64[0, 1, 2, 3, 2], 2)) + @test Parquet._decodedictionaryindices(encoded, 5, 1, Parquet.Limits()) == + (UInt64[0, 1, 2, 3, 2], length(encoded) + 1) + @test Parquet._decodedictionaryindices(UInt8[], 0, 1, Parquet.Limits()) == (UInt64[], 1) + @test_throws Parquet.FormatError Parquet._decodedictionaryindices(UInt8[33], 1, 1, Parquet.Limits()) + dictionary = Parquet.DecodedDictionary([UInt8[0x01]]) + values = Parquet._lookupdictionary(dictionary, UInt64[0, 0]) + values[1][1] = 0xff + @test values[2] == UInt8[0x01] + @test_throws Parquet.FormatError Parquet._lookupdictionary(dictionary, UInt64[1]) + # Dictionary entries deduplicate by exact bits, so +0.0 and -0.0 stay distinct + # and two NaN payloads are not collapsed. `_dictionaryentries` is the shared + # entry point used by the chunk writer. + bits = UInt32[0x00000000, 0x80000000, 0x7fc00001, 0x7fc00002] + floats = reinterpret(Float32, bits) + floatvalues, _ = Parquet._dictionaryentries( + Parquet._writecolumn(:value, repeat(collect(floats), 8))) + @test reinterpret(UInt32, floatvalues) == bits + raw = Vector{UInt8}[UInt8[1, 2], UInt8[1, 2], UInt8[3], UInt8[1, 2]] + rawvalues, rawindices = Parquet._dictionaryentries( + Parquet._writecolumn(:value, raw)) + @test rawvalues == Vector{UInt8}[UInt8[1, 2], UInt8[3]] + @test rawindices == UInt64[0, 0, 1, 0] +end + +@testset "dictionary V1 decoding" begin + int32 = columnleaf(MD.Type.INT32) + dictionary = dictionarypage(Int32[10, 20, 30]) + data = dictionarydatapage(UInt64[2, 0, 1, 2]) + @test readdictionarypages(dictionary, data, int32; num_values=4) == Int32[30, 10, 20, 30] + legacy = dictionarypage(Int32[10, 20, 30]; encoding=MD.Encoding.PLAIN_DICTIONARY) + legacydata = dictionarydatapage(UInt64[1, 2, 0]; encoding=MD.Encoding.PLAIN_DICTIONARY) + @test readdictionarypages(legacy, legacydata, int32; num_values=3) == Int32[20, 30, 10] + # parquet-mr 1.10 can omit the dictionary offset and point the data offset at it. + @test readdictionarypages(legacy, legacydata, int32; num_values=3, + data_page_offset=4, dictionary_page_offset=nothing) == + Int32[20, 30, 10] + # Modern footer offsets identify both frames exactly. + @test readdictionarypages(legacy, legacydata, int32; num_values=3, + data_page_offset=4 + length(legacy), dictionary_page_offset=Int64(4)) == + Int32[20, 30, 10] + optional = columnleaf(MD.Type.INT32; repetition=MD.FieldRepetitionType.OPTIONAL) + optionaldata = dictionarydatapage(UInt64[1, 0, 1]; levels=[1, 0, 1, 1], maxlevel=1) + @test isequal(readdictionarypages(dictionary, optionaldata, optional; num_values=4), + Union{Missing,Int32}[20, missing, 10, 20]) + allnull = dictionarydatapage(UInt64[]; levels=[0, 0, 0], maxlevel=1, omit_indices=true) + @test isequal(readdictionarypages(dictionary, allnull, optional; num_values=3), + Union{Missing,Int32}[missing, missing, missing]) + booldictionary = dictionarypage(Bool[false, true]) + booldata = dictionarydatapage(UInt64[1, 0, 1, 1]) + @test readdictionarypages(booldictionary, booldata, columnleaf(MD.Type.BOOLEAN); + num_values=4) == Bool[true, false, true, true] + bytesdictionary = dictionarypage(Vector{UInt8}[UInt8[0x61], UInt8[0x62, 0x63]]) + bytesdata = dictionarydatapage(UInt64[1, 0, 1]) + bytes = readdictionarypages(bytesdictionary, bytesdata, columnleaf(MD.Type.BYTE_ARRAY); + num_values=3) + @test bytes == Vector{UInt8}[UInt8[0x62, 0x63], UInt8[0x61], UInt8[0x62, 0x63]] + bytes[1][1] = 0xff + @test bytes[3] == UInt8[0x62, 0x63] + fixedvalues = Vector{UInt8}[UInt8[1, 2], UInt8[3, 4]] + fixeddictionary = dictionarypage(fixedvalues; width=2) + fixeddata = dictionarydatapage(UInt64[1, 0, 1]) + @test readdictionarypages(fixeddictionary, fixeddata, + columnleaf(MD.Type.FIXED_LEN_BYTE_ARRAY; width=2); num_values=3) == + Vector{UInt8}[UInt8[3, 4], UInt8[1, 2], UInt8[3, 4]] + first = dictionarydatapage(UInt64[0, 1]) + fallback = datapage(Int32[30, 40]) + @test readdictionarypages(dictionary, [first, fallback], int32; num_values=4) == + Int32[10, 20, 30, 40] + emptydictionary = dictionarypage(Int32[]) + @test readdictionarypages(emptydictionary, Vector{UInt8}[], int32; num_values=0, + data_page_offset=0) == Int32[] +end + +@testset "dictionary malformed input" begin + int32 = columnleaf(MD.Type.INT32) + dictionary = dictionarypage(Int32[10, 20]) + data = dictionarydatapage(UInt64[0, 1]) + @test_throws Parquet.FormatError readsynthetic([data], int32; num_values=2) + outofrange = dictionarydatapage(UInt64[0, 2]) + @test_throws Parquet.FormatError readdictionarypages(dictionary, outofrange, int32; num_values=2) + wide = dictionarydatapage(UInt64[0]; bitwidth=33) + @test_throws Parquet.FormatError readdictionarypages(dictionary, wide, int32; num_values=1) + trailing = dictionarydatapage(UInt64[0]; extra=UInt8[0x00]) + @test_throws Parquet.FormatError readdictionarypages(dictionary, trailing, int32; num_values=1) + baddictionary = dictionarypage(Int32[10]; extra=UInt8[0x00]) + @test_throws Parquet.FormatError readdictionarypages(baddictionary, + dictionarydatapage(UInt64[0]), int32; num_values=1) + wrongencoding = dictionarypage(Int32[10]; encoding=MD.Encoding.DELTA_BINARY_PACKED) + @test_throws Parquet.FormatError readdictionarypages(wrongencoding, + dictionarydatapage(UInt64[0]), int32; num_values=1) + negative = dictionarypage(Int32[]; count=-1) + @test_throws Parquet.FormatError readdictionarypages(negative, Vector{UInt8}[], int32; + num_values=0, data_page_offset=0) + truncated = dictionarypage(Int32[10]; count=2) + @test_throws Parquet.FormatError readdictionarypages(truncated, + dictionarydatapage(UInt64[0]), int32; num_values=1) + emptydictionary = dictionarypage(Int32[]) + @test_throws Parquet.FormatError readdictionarypages(emptydictionary, + dictionarydatapage(UInt64[0]), int32; num_values=1) + @test_throws Parquet.FormatError readsynthetic([datapage(Int32[1]), dictionary, data], int32; + num_values=3, dictionary_page_offset=4 + length(datapage(Int32[1])), data_page_offset=4) + @test_throws Parquet.FormatError readdictionarypages(dictionary, [dictionary, data], int32; + num_values=2) + index = columnpage(UInt8[]; type=MD.PageType.INDEX_PAGE, + index=MD.IndexPageHeader(), crc=:none) + @test_throws Parquet.FormatError readsynthetic([index, dictionary, data], int32; + num_values=2, index_page_offset=Int64(4), + dictionary_page_offset=nothing, data_page_offset=4 + length(index)) + unknown = columnpage(UInt8[]; type=MD.PageType.T(99), crc=:none) + @test readsynthetic([dictionary, unknown, data], int32; num_values=2, + dictionary_page_offset=4, data_page_offset=4 + length(dictionary) + length(unknown)) == + Int32[10, 20] + corrupt = dictionarypage(Int32[10]; crc=Int32(0)) + @test_throws Parquet.FormatError readdictionarypages(corrupt, + dictionarydatapage(UInt64[0]), int32; num_values=1) + @test_throws Parquet.LimitError readdictionarypages(dictionary, data, int32; + num_values=2, limits=Parquet.Limits(max_container_elements=1)) +end + +@testset "adaptive dictionary writer" begin + rawvalue = UInt8[0x00, 0xff, 0x41] + floatpattern = Float64[0.0, -0.0, reinterpret(Float64, UInt64(0x7ff8000000000001)), + reinterpret(Float64, UInt64(0x7ff8000000000002))] + input = ( + integers=fill(Int32(42), 128), + floats=repeat(floatpattern, 32), + strings=fill("repeated string", 128), + raw=fill(rawvalue, 128), + optional=Union{Missing,Int64}[isodd(index) ? 7 : missing for index in 1:128], + flags=repeat(Bool[true, false], 64), + ) + bytes = Parquet._encodefile(input; dictionary=true) + @test bytes == Parquet._encodefile(input; dictionary=true) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + chunks = metadata.row_groups[1].columns + @test all(chunks[index].meta_data.dictionary_page_offset !== nothing for index in 1:5) + @test chunks[6].meta_data.dictionary_page_offset === nothing + @test all(MD.Encoding.RLE_DICTIONARY in chunks[index].meta_data.encodings for index in 1:5) + @test chunks[6].meta_data.encodings == [MD.Encoding.PLAIN] + @test chunks[1].meta_data.encoding_stats == [ + MD.PageEncodingStats(page_type=MD.PageType.DICTIONARY_PAGE, + encoding=MD.Encoding.PLAIN, count=Int32(1)), + MD.PageEncodingStats(page_type=MD.PageType.DATA_PAGE, + encoding=MD.Encoding.RLE_DICTIONARY, count=Int32(1)), + ] + close(file) + table = Parquet.Table(bytes) + @test table.columns.integers == input.integers + @test reinterpret(UInt64, table.columns.floats) == reinterpret(UInt64, input.floats) + @test table.columns.strings == input.strings + @test table.columns.raw == input.raw + @test isequal(table.columns.optional, input.optional) + @test table.columns.flags == input.flags + table.columns.raw[1][1] = 0xaa + @test table.columns.raw[2] == rawvalue + close(table) + boolean = Parquet._encodefile((value=fill(true, 2000),); dictionary=true) + booleanfile = Parquet.File(boolean) + booleanmeta = TH.decode(booleanfile.footer.bytes, MD.FileMetaData) + booleanchunk = booleanmeta.row_groups[1].columns[1].meta_data + @test booleanchunk.dictionary_page_offset === nothing + @test booleanchunk.encodings == [MD.Encoding.PLAIN] + close(booleanfile) + booleantable = Parquet.Table(boolean) + @test booleantable.columns.value == fill(true, 2000) + close(booleantable) + allmissing = Union{Missing,Int32}[missing for _ in 1:8] + missingcolumn = Parquet._writecolumn(:value, allmissing) + missinglimit = length(Parquet._definitionpayload(missingcolumn)) + tight = Parquet.Limits(max_page_bytes=missinglimit) + @test Parquet._encodefile((value=allmissing,); dictionary=true, limits=tight) == + Parquet._encodefile((value=allmissing,); dictionary=false, limits=tight) + plain = Parquet._encodefile((value=fill(Int32(1), 128),)) + plainfile = Parquet.File(plain) + plainmeta = TH.decode(plainfile.footer.bytes, MD.FileMetaData) + @test plainmeta.row_groups[1].columns[1].meta_data.dictionary_page_offset === nothing + close(plainfile) + unique = Parquet._encodefile((value=collect(Int32(1):Int32(32)),); dictionary=true) + uniquefile = Parquet.File(unique) + uniquemeta = TH.decode(uniquefile.footer.bytes, MD.FileMetaData) + @test uniquemeta.row_groups[1].columns[1].meta_data.dictionary_page_offset === nothing + close(uniquefile) + io = IOBuffer() + Parquet.write(io, (value=fill(Int64(9), 100),); dictionary=true, checksum=false) + @test Parquet.Table(take!(io)).columns.value == fill(Int64(9), 100) +end + +@testset "adaptive dictionary V2 writer" begin + input = (value=fill(Int32(42), 128),) + dictionarycount = 0 + plaincount = 0 + for codec in (:uncompressed, :snappy, :gzip, :brotli, :zstd, :lz4_raw) + bytes = Parquet._encodefile(input; dictionary=true, codec=codec, pageversion=:v2) + plain = Parquet._encodefile(input; dictionary=false, codec=codec, pageversion=:v2) + table = Parquet.Table(bytes) + @test table.columns.value == input.value + close(table) + pages, metadata = writtenpages(bytes, 1) + _, plainmetadata = writtenpages(plain, 1) + @test metadata.total_compressed_size <= plainmetadata.total_compressed_size + if metadata.dictionary_page_offset === nothing + plaincount += 1 + @test [page.header.type_ for page in pages] == [MD.PageType.DATA_PAGE_V2] + @test pages[1].header.data_page_header_v2.encoding == MD.Encoding.PLAIN + else + dictionarycount += 1 + @test [page.header.type_ for page in pages] == + [MD.PageType.DICTIONARY_PAGE, MD.PageType.DATA_PAGE_V2] + @test pages[2].header.data_page_header_v2.encoding == MD.Encoding.RLE_DICTIONARY + @test metadata.encoding_stats == [ + MD.PageEncodingStats(page_type=MD.PageType.DICTIONARY_PAGE, + encoding=MD.Encoding.PLAIN, count=Int32(1)), + MD.PageEncodingStats(page_type=MD.PageType.DATA_PAGE_V2, + encoding=MD.Encoding.RLE_DICTIONARY, count=Int32(1)), + ] + end + end + @test dictionarycount > 0 + @test plaincount > 0 +end + +@testset "official dictionary fixtures" begin + if !isdir(columncorpus()) + @warn "parquet-testing corpus not found; skipping dictionary corpus tests" COLUMN_CORPUS + else + longs, _ = corpuscolumn(columncorpus("plain-dict-uncompressed-checksum.parquet"), 1) + binary, _ = corpuscolumn(columncorpus("plain-dict-uncompressed-checksum.parquet"), 2) + expected = collect(codeunits("a655fd0e-9949-4059-bcae-fd6a002a4652")) + @test longs == zeros(Int64, 1000) + @test length(binary) == 1000 && all(==(expected), binary) + @test binary[1] !== binary[2] + @test_throws Parquet.FormatError corpuscolumn( + columncorpus("rle-dict-uncompressed-corrupt-checksum.parquet"), 1) + indexed = Parquet.Table(columncorpus("data_index_bloom_encoding_with_length.parquet")) + @test indexed.columns.String == ["Hello", "This is", "a", "test", "How", "are you", + "doing ", "today", "the quick", "brown fox", "jumps", "over", "the lazy", "dog"] + close(indexed) + alltypes = Parquet.File(columncorpus("alltypes_dictionary.parquet")) + allmeta = TH.decode(alltypes.footer.bytes, MD.FileMetaData) + allschema = Parquet.Schema(allmeta) + @test Parquet.readcolumn(alltypes, allmeta, allschema, 1, 1) == Int32[0, 1] + @test Parquet.readcolumn(alltypes, allmeta, allschema, 1, 2) == Bool[true, false] + @test Parquet.readcolumn(alltypes, allmeta, allschema, 1, 6) == Int64[0, 10] + @test Parquet.readcolumn(alltypes, allmeta, allschema, 1, 9) == + Vector{UInt8}[collect(codeunits("01/01/09")), collect(codeunits("01/01/09"))] + close(alltypes) + tiny = Parquet.File(columncorpus("alltypes_tiny_pages.parquet")) + tinymeta = TH.decode(tiny.footer.bytes, MD.FileMetaData) + tinyschema = Parquet.Schema(tinymeta) + tinyints = Parquet.readcolumn(tiny, tinymeta, tinyschema, 1, 3) + tinystrings = Parquet.readcolumn(tiny, tinymeta, tinyschema, 1, 10) + @test length(tinyints) == 7300 && sum(Int64, tinyints) == 32850 && extrema(tinyints) == (0, 9) + @test tinyints[1:12] == Int32[2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3] + @test length(unique(tinystrings)) == 10 + @test String.(tinystrings[1:12]) == ["2", "3", "4", "5", "6", "7", "8", "9", "0", "1", "2", "3"] + @test tinymeta.row_groups[1].columns[3].meta_data.dictionary_page_offset === nothing + close(tiny) + empty = Parquet.Table(columncorpus("column_chunk_key_value_metadata.parquet")) + @test length(empty) == 0 + close(empty) + end +end diff --git a/test/footer.jl b/test/footer.jl new file mode 100644 index 0000000..877ba6d --- /dev/null +++ b/test/footer.jl @@ -0,0 +1,223 @@ +function parquetbytes(footer::Vector{UInt8}; encrypted::Bool=false) + magic = encrypted ? UInt8[0x50, 0x41, 0x52, 0x45] : UInt8[0x50, 0x41, 0x52, 0x31] + lengthbytes = reinterpret(UInt8, [htol(UInt32(length(footer)))]) + return vcat(magic, footer, lengthbytes, magic) +end + +struct TestSource <: Parquet.AbstractSource + bytes::Vector{UInt8} +end + +function Parquet.sourcelength(src::TestSource) + return Int64(length(src.bytes)) +end + +function Parquet.readrange(src::TestSource, offset::Integer, count::Integer) + first = Int(offset) + 1 + return @view src.bytes[first:(first + Int(count) - 1)] +end + +struct ShiftedFooterBytes <: AbstractVector{UInt8} + bytes::Vector{UInt8} +end + +function Base.IndexStyle(::Type{ShiftedFooterBytes}) + return IndexLinear() +end + +function Base.size(bytes::ShiftedFooterBytes) + return (length(bytes.bytes),) +end + +function Base.axes(bytes::ShiftedFooterBytes) + return (2:(length(bytes.bytes) + 1),) +end + +function Base.getindex(bytes::ShiftedFooterBytes, index::Int) + checkbounds(bytes, index) + return bytes.bytes[index - 1] +end + +mutable struct FooterCallbackSentinel <: Exception + id::Int +end + +mutable struct FooterProbeSource <: Parquet.AbstractSource + bytes::Vector{UInt8} + mode::Symbol + faultread::Int + lengthcalls::Int + reads::Vector{Tuple{Int64,Int64}} + closes::Int + sentinel::Union{Nothing,FooterCallbackSentinel} +end + + +function FooterProbeSource(bytes::Vector{UInt8}; mode::Symbol=:normal, + faultread::Int=0, sentinel=nothing) + return FooterProbeSource(bytes, mode, faultread, 0, + Tuple{Int64,Int64}[], 0, sentinel) +end + +function Parquet.sourcelength(source::FooterProbeSource) + source.lengthcalls += 1 + source.mode === :length_throw && throw(something(source.sentinel)) + source.mode === :length_type && return Float64(length(source.bytes)) + source.mode === :length_negative && return Int64(-1) + source.mode === :length_changing && source.lengthcalls > 1 && return Int64(0) + return Int64(length(source.bytes)) +end + +function Parquet.readrange(source::FooterProbeSource, offset::Integer, + count::Integer) + offset64 = Int64(offset) + count64 = Int64(count) + push!(source.reads, (offset64, count64)) + if length(source.reads) == source.faultread + source.mode === :throw && throw(something(source.sentinel)) + source.mode === :short && return fill(UInt8(0), max(Int(count64) - 1, 0)) + source.mode === :long && return fill(UInt8(0), Int(count64) + 1) + source.mode === :wrong_type && return fill(Int8(0), Int(count64)) + source.mode === :wrong_axes && return ShiftedFooterBytes( + fill(UInt8(0), Int(count64))) + end + first = Int(offset64) + 1 + return @view source.bytes[first:(first + Int(count64) - 1)] +end + +function Parquet.close!(source::FooterProbeSource) + source.closes += 1 + return +end + +@testset "footer framing" begin + input = parquetbytes(UInt8[0x01, 0x02, 0x03]) + file = Parquet.File(input) + @test file.footer.offset == 4 + @test file.footer.length == 3 + @test !file.footer.encrypted + @test collect(file.footer.bytes) == UInt8[0x01, 0x02, 0x03] + close(file) + close(file) + + encrypted = Parquet.File(parquetbytes(UInt8[0xaa]; encrypted=true)) + @test encrypted.footer.encrypted + close(encrypted) + + custom = Parquet.File(TestSource(input)) + @test custom.footer.length == 3 + @test !Parquet.concurrentreads(custom.source) + close(custom) + + @test_throws Parquet.FormatError Parquet.File(UInt8[]) + @test_throws Parquet.FormatError Parquet.File(vcat(UInt8[0x00, 0x00, 0x00, 0x00], input[5:end])) + @test_throws Parquet.LimitError Parquet.File(input; limits=Parquet.Limits(max_footer_bytes=2)) + + invalidlength = copy(input) + invalidlength[(end - 7):(end - 4)] .= 0xff + @test_throws Parquet.FormatError Parquet.File(invalidlength) + + oversizefooter = copy(input) + oversizefooter[(end - 7):(end - 4)] .= UInt8[0x04, 0x00, 0x00, 0x00] + @test_throws Parquet.FormatError Parquet.File(oversizefooter) +end + + +@testset "footer exact source reads" begin + input = parquetbytes(UInt8[0x01, 0x02, 0x03]) + source = FooterProbeSource(input; mode=:length_changing) + file = Parquet.File(source) + @test source.lengthcalls == 1 + @test source.reads == [(Int64(0), Int64(4)), + (Int64(length(input) - 8), Int64(8)), (Int64(4), Int64(3))] + close(file) + @test source.closes == 1 + + invalid = copy(input) + invalid[1] = 0x00 + invalidsource = FooterProbeSource(invalid) + @test_throws Parquet.FormatError Parquet.File(invalidsource) + @test invalidsource.lengthcalls == 1 + @test invalidsource.reads == [(Int64(0), Int64(4))] + @test invalidsource.closes == 0 + + for mode in (:short, :long, :wrong_type, :wrong_axes) + for faultread in 1:3 + malformed = FooterProbeSource(input; mode=mode, + faultread=faultread) + @test_throws ArgumentError Parquet.File(malformed) + @test length(malformed.reads) == faultread + @test malformed.closes == 0 + end + end + + for faultread in 1:3 + sentinel = FooterCallbackSentinel(faultread) + throwing = FooterProbeSource(input; mode=:throw, + faultread=faultread, sentinel=sentinel) + error = try + Parquet.File(throwing) + nothing + catch err + err + end + @test error === sentinel + @test length(throwing.reads) == faultread + @test throwing.closes == 0 + end + + for mode in (:length_type, :length_negative) + invalidlength = FooterProbeSource(input; mode=mode) + @test_throws ArgumentError Parquet.File(invalidlength) + @test invalidlength.lengthcalls == 1 + @test isempty(invalidlength.reads) + @test invalidlength.closes == 0 + end + sentinel = FooterCallbackSentinel(4) + throwinglength = FooterProbeSource(input; mode=:length_throw, + sentinel=sentinel) + error = try + Parquet.File(throwinglength) + nothing + catch err + err + end + @test error === sentinel + @test isempty(throwinglength.reads) + @test throwinglength.closes == 0 +end + +@testset "File source ownership and copied IO budget" begin + input = parquetbytes(UInt8[0x01, 0x02, 0x03]) + failed = FooterProbeSource(UInt8[]) + @test_throws Parquet.FormatError Parquet.File(failed) + @test failed.closes == 0 + + adopted = FooterProbeSource(input) + file = Parquet.File(adopted) + @test adopted.closes == 0 + close(file) + close(file) + @test adopted.closes == 1 + + tablefailure = FooterProbeSource(input) + @test_throws Parquet.FormatError Parquet.Table(tablefailure) + @test tablefailure.closes == 1 + + limits = Parquet.Limits(max_materialized_bytes=100_000) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, Int64(23)) + invalidio = IOBuffer(UInt8[]) + @test_throws Parquet.FormatError Parquet.File(invalidio; limits=limits, + budget=budget) + @test Parquet._budgetused(budget) == 23 + @test isopen(invalidio) + + validio = IOBuffer(input) + copied = Parquet.File(validio; limits=limits, budget=budget) + @test Parquet._budgetused(budget) == + 23 + Parquet._materializedarraybytes(UInt8, length(input)) + close(copied) + @test Parquet._budgetused(budget) == 23 + @test isopen(validio) +end diff --git a/test/generate_artifacts.jl b/test/generate_artifacts.jl deleted file mode 100644 index ac15da7..0000000 --- a/test/generate_artifacts.jl +++ /dev/null @@ -1,17 +0,0 @@ -using ArtifactUtils, Artifacts - -add_artifact!( - joinpath(@__DIR__, "..", "Artifacts.toml"), - "parcompat", - "https://github.com/Parquet/parquet-compatibility/archive/2b47eac447c7a4a88247651a4065984db7b247ff.tar.gz", - force=true, - lazy=true, -) - -add_artifact!( - joinpath(@__DIR__, "..", "Artifacts.toml"), - "julia_parcompat", - "https://github.com/JuliaIO/parquet-compatibility/archive/3f7586f1b7f2a0c6b048791fb5f97c0b3df52e39.tar.gz", - force=true, - lazy=true, -) diff --git a/test/generator.jl b/test/generator.jl new file mode 100644 index 0000000..3153599 --- /dev/null +++ b/test/generator.jl @@ -0,0 +1,167 @@ +include(joinpath(@__DIR__, "..", "thrift", "generate.jl")) +using SHA + +if !@isdefined(TH) + const TH = Parquet.Thrift +end + +const GENERATOR_TEST_IDL = """ +namespace jl ParquetTest + +enum Color { + RED = 1; + GREEN = 2, + BLUE = 4 +} + +struct Inner { + 1: required i32 a + 2: optional string b; +} + +union Choice { + 1: Inner one + 2: i64 two +} + +struct Defaults { + 1: required i64 large = 7 + 2: optional bool is_compressed = true +} + +struct Empty {} + +struct Both { + 1: optional Inner one + 2: optional i64 two +} + +struct Other { + 3: optional i32 three +} + +struct Mixed { + 2: optional i64 two + 3: optional i32 three +} + +struct TwoUnknown { + 3: optional i32 three + 4: optional i32 four +} + +struct Bits { + 1: required list bits +} + +struct Painted { + 1: optional Color color +} + +struct Everything { + 1: required bool flag + 2: optional bool maybe + 3: required byte small + 4: required i16 medium + 5: required i32 type + 6: required i64 large + 7: required double real + 8: required string text + 9: required binary blob + 10: required list ints + 11: optional set names + 12: optional map counts + 13: optional list> matrix + 14: optional Color color + 15: optional Inner inner + 16: optional list inners + 17: optional Choice choice + 18: optional map byid + 19: optional list> maps +} +""" + +function generatedmodule(idl::String) + mod = Module(:GeneratedThriftTest) + Core.eval(mod, Expr(:const, Expr(:(=), :Thrift, TH))) + Base.include_string(mod, ThriftGenerator.generate(idl; version="test", commit="none")) + return Base.invokelatest(getfield, mod, :Metadata) +end + +@testset "generator regeneration is deterministic and current" begin + idlbytes = read(ThriftGenerator.IDL_PATH) + gitblob = vcat(codeunits("blob $(length(idlbytes))\0"), idlbytes) + @test bytes2hex(sha1(gitblob)) == "fe259d61bc470ade78bad48f5223a82598b91b59" + @test bytes2hex(sha256(idlbytes)) == "53bb8fc9b96469d7ca694121ead839e449e5156d7bf79f0df728cdd72796df38" + idl = String(idlbytes) + generated = ThriftGenerator.generate(idl) + @test generated == ThriftGenerator.generate(idl) + @test generated == read(ThriftGenerator.OUTPUT_PATH, String) + @test occursin("apache/parquet-format 2.13.0 (c47e2a66e88943fc46fde1b028a9432f14fdf5c0)", generated) + @test occursin("# 1: optional Type type\n", generated) + @test occursin("type_::Union{Nothing, Type.T}", generated) + @test occursin("# 7: optional bool is_compressed = true\n", generated) + @test !occursin("Dict{Symbol", generated) + @test ThriftGenerator.main(["--check"]) == 0 + mangled = ThriftGenerator.generate("struct A { 1: required i32 end }") + @test occursin(" end_::Int32 # 1: required i32 end\n", mangled) + @test_throws ErrorException ThriftGenerator.generate("struct A { 1: required Missing a }") + @test_throws ErrorException ThriftGenerator.generate("struct A { 1: required i32 a 1: required i32 b }") + @test_throws ErrorException ThriftGenerator.generate("struct A { 1: required list> a }") + @test_throws ErrorException ThriftGenerator.generate("service A {}") +end + +@testset "generated code covers every Thrift type" begin + G = generatedmodule(GENERATOR_TEST_IDL) + x = G.Everything(flag=true, small=Int8(-3), medium=Int16(-300), type_=Int32(5), large=Int64(1) << 40, real=2.5, + text="héllo", blob=UInt8[1, 2, 3], ints=Int32[1, -2, 3], names=["a", "b"], counts=["k" => Int64(1)], + matrix=[[true, false], Bool[]], color=G.Color.BLUE, inner=G.Inner(a=Int32(1)), + inners=[G.Inner(a=Int32(2), b="x")], choice=G.Choice(two=Int64(9)), byid=[Int32(1) => G.Inner(a=Int32(3))], + maps=[["m" => Int32(4)], Pair{String,Int32}[]]) + bytes = TH.encode(x) + y = TH.decode(bytes, G.Everything) + @test isequal(x, y) && x == y && hash(x) == hash(y) + @test TH.encode(y) == bytes + @test y.maybe === nothing && y.type_ == 5 && y.names == ["a", "b"] && y.choice.two == 9 + @test fieldnames(G.Everything)[5] == :type_ + @test_throws UndefKeywordError G.Everything(flag=true) + @test TH.decode(TH.encode(G.Empty()), G.Defaults) == G.Defaults(large=Int64(7), is_compressed=nothing) + @test G.Defaults().large == 7 && G.Defaults().is_compressed === nothing + @test TH.decode(UInt8[0x00], G.Defaults).large == 7 + @test TH.decode(TH.encode(G.Defaults(is_compressed=false)), G.Defaults).is_compressed === false + @test_throws Parquet.FormatError TH.decode(UInt8[0x00], G.Inner) + @test_throws UndefKeywordError G.Inner() + @test G.Color.RED.value == 1 && G.Color.BLUE.value == 4 + @test TH.name(G.Color.GREEN) === :GREEN && TH.name(G.Color.T(9)) === nothing + @test sprint(show, G.Color.RED) == "Color.RED" && sprint(show, G.Color.T(9)) == "Color.T(9)" + @test TH.decode(TH.encode(G.Painted(color=G.Color.T(9))), G.Painted).color == G.Color.T(9) + @test_throws InexactError G.Color.T(Int64(2)^40) + bits = TH.encode(G.Bits(bits=[true])) + @test bits == UInt8[0x19, 0x11, 0x01, 0x00] + bits[3] = 0x00 + @test_throws Parquet.FormatError TH.decode(bits, G.Bits) +end + +@testset "generated unions" begin + G = generatedmodule(GENERATOR_TEST_IDL) + @test G.Choice(two=Int64(1)).two == 1 + @test_throws ArgumentError G.Choice(one=G.Inner(a=Int32(1)), two=Int64(2)) + @test_throws ArgumentError G.Choice(two=Int64(2), unknown_fields=(TH.RawField(3, TH.I32, UInt8[0x02]),)) + @test_throws ArgumentError G.Choice(unknown_fields=(TH.RawField(3, TH.I32, UInt8[0x02]), TH.RawField(4, TH.I32, UInt8[0x02]))) + @test G.Choice() == G.Choice() && TH.decode(TH.encode(G.Choice()), G.Choice) == G.Choice() + both = TH.encode(G.Both(one=G.Inner(a=Int32(1)), two=Int64(2))) + @test_throws Parquet.FormatError TH.decode(both, G.Choice) + other = TH.encode(G.Other(three=Int32(5))) + choice = TH.decode(other, G.Choice) + @test choice.one === nothing && choice.two === nothing + @test length(choice.unknown_fields) == 1 && choice.unknown_fields[1].id == 3 + @test TH.encode(choice) == other + mutablechoice = G.Choice(two=Int64(2)) + push!(mutablechoice.unknown_fields, TH.RawField(3, TH.I32, UInt8[0x02])) + @test_throws ArgumentError TH.encode(mutablechoice) + @test_throws Parquet.FormatError TH.decode(TH.encode(G.Mixed(two=Int64(1), three=Int32(2))), G.Choice) + @test_throws Parquet.FormatError TH.decode(TH.encode(G.TwoUnknown(three=Int32(1), four=Int32(2))), G.Choice) + wrapped = G.Everything(flag=false, small=Int8(0), medium=Int16(0), type_=Int32(0), large=Int64(0), real=0.0, + text="", blob=UInt8[], ints=Int32[], choice=choice) + @test TH.decode(TH.encode(wrapped), G.Everything).choice == choice +end diff --git a/test/limits.jl b/test/limits.jl new file mode 100644 index 0000000..a2d0b1a --- /dev/null +++ b/test/limits.jl @@ -0,0 +1,252 @@ +@testset "shared live-byte budget" begin + limits = Parquet.Limits(max_materialized_bytes=10) + budget = Parquet._LiveByteBudget(limits) + @test Parquet._budgetused(budget) == 0 + Parquet._reserve!(budget, 4) + @test Parquet._budgetused(budget) == 4 + @test_throws Parquet.LimitError Parquet._reserve!(budget, 7) + @test Parquet._budgetused(budget) == 4 + Parquet._release!(budget, 3) + @test Parquet._budgetused(budget) == 1 + @test_throws ArgumentError Parquet._reserve!(budget, -1) + @test_throws ArgumentError Parquet._release!(budget, 2) + @test Parquet._budgetused(budget) == 1 +end + +@testset "bounded schema-name interning" begin + before = Parquet._internedschemanamebytes() + name = "__parquet_schema_name_budget_regression__" + charge = Parquet._schemanamecharge(name, typemax(Int64)) + small = Parquet.Limits(max_schema_name_bytes=charge - 1) + @test_throws Parquet.LimitError Parquet._internschemanames(String[name], small) + @test Parquet._internedschemanamebytes() == before + + exact = Parquet.Limits(max_schema_name_bytes=charge) + @test Parquet._internschemanames(String[name], exact) == Symbol[Symbol(name)] + @test Parquet._internedschemanamebytes() == before + charge + @test Parquet._internschemanames(String[name], + Parquet.Limits(max_schema_name_bytes=0)) == Symbol[Symbol(name)] + @test Parquet._internedschemanamebytes() == before + charge + + # Regression: the limit admits each operation's new names independently, so + # earlier interning (hostile or not) must not consume later operations' budget. + second = "__parquet_schema_name_budget_regression_second__" + secondcharge = Parquet._schemanamecharge(second, typemax(Int64)) + @test Parquet._internschemanames(String[second], + Parquet.Limits(max_schema_name_bytes=secondcharge)) == Symbol[Symbol(second)] + @test Parquet._internedschemanamebytes() == before + charge + secondcharge + + @test_throws Parquet.UnsupportedFeatureError Parquet._internschemanames( + String["duplicate", "duplicate"], Parquet.Limits()) + @test_throws Parquet.UnsupportedFeatureError Parquet._internschemanames( + String["nul\0name"], Parquet.Limits()) + @test Parquet._internedschemanamebytes() == before + charge + secondcharge +end + +@testset "isolated schema-name boundary and precedence" begin + project = dirname(something(Base.active_project())) + script = raw""" + using Parquet + Parquet._internedschemanamebytes() == 0 || exit(10) + name = "__parquet_isolated_exact_schema_name__" + charge = Parquet._schemanamecharge(name, typemax(Int64)) + try + Parquet._internschemanames(String[name], + Parquet.Limits(max_schema_name_bytes=charge - 1)) + exit(11) + catch err + err isa Parquet.LimitError || exit(12) + end + Parquet._internedschemanamebytes() == 0 || exit(13) + for invalid in (String["duplicate", "duplicate"], + String["valid", "nul\0name"]) + try + Parquet._internschemanames(invalid, + Parquet.Limits(max_schema_name_bytes=0)) + exit(14) + catch err + err isa Parquet.UnsupportedFeatureError || exit(15) + end + Parquet._internedschemanamebytes() == 0 || exit(16) + end + exact = Parquet.Limits(max_schema_name_bytes=charge) + budget = Parquet._LiveByteBudget(exact) + Parquet._reserve!(budget, Int64(64)) + Parquet._internschemanames(String[name], exact, budget) == + Symbol[Symbol(name)] || exit(17) + Parquet._internedschemanamebytes() == charge || exit(18) + expected = Int64(64) + + Parquet._materializedarraybytes(Symbol, 1) + Parquet._budgetused(budget) == expected || exit(19) + second = "__parquet_isolated_exact_schema_name_second__" + secondcharge = Parquet._schemanamecharge(second, typemax(Int64)) + Parquet._internschemanames(String[second], + Parquet.Limits(max_schema_name_bytes=secondcharge)) == + Symbol[Symbol(second)] || exit(20) + Parquet._internedschemanamebytes() == charge + secondcharge || exit(21) + """ + command = `$(Base.julia_cmd()) --startup-file=no --project=$project -e $script` + @test success(command) +end + +function minimummaterializedlimit(f; maximum::Int64=1_000_000) + low = Int64(-1) + high = maximum + f(Parquet.Limits(max_materialized_bytes=high)) + while high - low > 1 + middle = (low + high) ÷ 2 + try + f(Parquet.Limits(max_materialized_bytes=middle)) + high = middle + catch err + err isa Parquet.LimitError || rethrow() + low = middle + end + end + return high +end + +@testset "operation materialization budgets" begin + one = (a=Int32[1, 2, 3, 4],) + two = (a=Int32[1, 2, 3, 4], b=Int32[5, 6, 7, 8]) + for maximum in (0, 1) + limits = Parquet.Limits(max_materialized_bytes=maximum) + @test_throws Parquet.LimitError Parquet._encodefile(one; limits=limits) + end + + onebytes = Parquet._encodefile(one) + twobytes = Parquet._encodefile(two) + for maximum in (0, 1) + limits = Parquet.Limits(max_materialized_bytes=maximum) + @test_throws Parquet.LimitError Parquet.Table(onebytes; limits=limits) + end + + file = Parquet.File(onebytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, + Parquet.Metadata.FileMetaData) + schema = Parquet.Schema(metadata) + close(file) + for maximum in (0, 1) + limits = Parquet.Limits(max_materialized_bytes=maximum) + @test_throws Parquet.LimitError Parquet._nestedplan(schema; limits=limits) + end + + writeone = minimummaterializedlimit() do limits + Parquet._encodefile(one; limits=limits) + return + end + writetwo = minimummaterializedlimit() do limits + Parquet._encodefile(two; limits=limits) + return + end + @test writetwo > writeone + @test_throws Parquet.LimitError Parquet._encodefile(two; + limits=Parquet.Limits(max_materialized_bytes=writeone)) + + readone = minimummaterializedlimit() do limits + table = Parquet.Table(onebytes; limits=limits) + close(table) + return + end + readtwo = minimummaterializedlimit() do limits + table = Parquet.Table(twobytes; limits=limits) + close(table) + return + end + @test readtwo > readone + @test_throws Parquet.LimitError Parquet.Table(twobytes; + limits=Parquet.Limits(max_materialized_bytes=readone)) + + tworowgroups = Parquet._encodefile((a=vcat(one.a, one.a),); + rowgroupsize=length(one.a)) + readtwogroups = minimummaterializedlimit() do limits + table = Parquet.Table(tworowgroups; limits=limits) + close(table) + return + end + @test readtwogroups > readone + @test_throws Parquet.LimitError Parquet.Table(tworowgroups; + limits=Parquet.Limits(max_materialized_bytes=readone)) +end + +@testset "footer metadata structural preflight" begin + MD = Parquet.Metadata + count = 2000 + schema = MD.SchemaElement[ + MD.SchemaElement(name="schema", num_children=Int32(count)), + ] + for _ in 1:count + push!(schema, MD.SchemaElement(name="", type_=MD.Type.INT32, + repetition_type=MD.FieldRepetitionType.REQUIRED)) + end + metadata = MD.FileMetaData(version=Int32(1), schema=schema, + num_rows=Int64(0), row_groups=MD.RowGroup[]) + footer = Parquet.Thrift.encode(metadata) + bytes = copy(Parquet.PARQUET_MAGIC) + append!(bytes, footer) + Parquet._writelittle!(bytes, UInt32(length(footer))) + append!(bytes, Parquet.PARQUET_MAGIC) + file = Parquet.File(bytes) + maximum = Int64(128 + 4 * length(footer)) + limits = Parquet.Limits(max_materialized_bytes=maximum, + max_container_elements=count + 10) + budget = Parquet._LiveByteBudget(limits) + @test_throws Parquet.LimitError Parquet._readfilemetadata(file, limits, + budget) + @test Parquet._budgetused(budget) == 0 + close(file) +end + +@testset "schema-name validation preflight" begin + before = Parquet._internedschemanamebytes() + first = "__parquet_new_name_rejected_before_set_growth__" + names = fill(first, 100_000) + limits = Parquet.Limits(max_schema_name_bytes=0, + max_materialized_bytes=1024 * 1024) + budget = Parquet._LiveByteBudget(limits) + @test_throws Parquet.UnsupportedFeatureError Parquet._internschemanames( + names, limits, budget) + @test Parquet._budgetused(budget) == 0 + @test Parquet._internedschemanamebytes() == before + + Parquet._reserve!(budget, Int64(64)) + for invalid in (String[first, first], String["nul\0name"]) + @test_throws Parquet.UnsupportedFeatureError Parquet._internschemanames( + invalid, limits, budget) + @test Parquet._budgetused(budget) == 64 + @test Parquet._internedschemanamebytes() == before + end + Parquet._release!(budget, Int64(64)) + + firstatomic = "__parquet_atomic_batch_first__" + secondatomic = "__parquet_atomic_batch_second__" + firstcharge = Parquet._schemanamecharge(firstatomic, typemax(Int64)) + atomiclimits = Parquet.Limits(max_schema_name_bytes=firstcharge) + atomicbudget = Parquet._LiveByteBudget(atomiclimits) + Parquet._reserve!(atomicbudget, Int64(64)) + @test_throws Parquet.LimitError Parquet._internschemanames( + String[firstatomic, secondatomic], atomiclimits, atomicbudget) + @test Parquet._budgetused(atomicbudget) == 64 + @test Parquet._internedschemanamebytes() == before + lock(Parquet._SCHEMA_NAME_REGISTRY.lock) + try + state = Parquet._SCHEMA_NAME_REGISTRY.state + @test !(firstatomic in state.names) + @test !(secondatomic in state.names) + finally + unlock(Parquet._SCHEMA_NAME_REGISTRY.lock) + end + + atomic = "__parquet_name_output_preflight_is_atomic__" + charge = Parquet._schemanamecharge(atomic, typemax(Int64)) + temporary = Parquet._materializedarraybytes(String, 0) + + 2 * Parquet._MATERIALIZED_OBJECT_BYTES + + Parquet._materializedarraybytes(String, 2; header=false) + tight = Parquet.Limits(max_schema_name_bytes=charge, + max_materialized_bytes=temporary) + tightbudget = Parquet._LiveByteBudget(tight) + @test_throws Parquet.LimitError Parquet._internschemanames(String[atomic], + tight, tightbudget) + @test Parquet._budgetused(tightbudget) == 0 + @test Parquet._internedschemanamebytes() == before +end diff --git a/test/logical.jl b/test/logical.jl new file mode 100644 index 0000000..59d39e5 --- /dev/null +++ b/test/logical.jl @@ -0,0 +1,150 @@ +using Dates + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end + +function logicaltestelement(name, physical; logical=nothing, converted=nothing) + return MD.SchemaElement(name=name, type_=physical, + repetition_type=MD.FieldRepetitionType.OPTIONAL, + logicalType=logical, converted_type=converted) +end + +@testset "logical annotation precedence and validation" begin + stringtype = MD.LogicalType(STRING=MD.StringType()) + datetype = MD.LogicalType(DATE=MD.DateType()) + modernstring = logicaltestelement("value", MD.Type.BYTE_ARRAY; + logical=stringtype, converted=MD.ConvertedType.DATE) + moderndate = logicaltestelement("value", MD.Type.INT32; + logical=datetype, converted=MD.ConvertedType.UTF8) + @test Parquet._logicalkind(modernstring) === :string + @test Parquet._logicalkind(moderndate) === :date + @test Parquet._logicaleltype(modernstring, Vector{UInt8}) === String + @test Parquet._logicaleltype(moderndate, Int32) === Date + + node = Parquet.SchemaNode(moderndate, ["value"], Int16(1), Int16(0), + Int32(1), Parquet.SchemaNode[]) + @test Parquet._logicalkind(node) === :date + @test Parquet._logicaleltype(node, Int32) === Date + + legacystring = logicaltestelement("value", MD.Type.BYTE_ARRAY; + converted=MD.ConvertedType.UTF8) + legacydate = logicaltestelement("value", MD.Type.INT32; + converted=MD.ConvertedType.DATE) + @test Parquet._logicalkind(legacystring) === :string + @test Parquet._logicalkind(legacydate) === :date + + unknown = MD.LogicalType(unknown_fields=(TH.RawField(2555, TH.STRUCT, UInt8[0x00]),)) + unknownmodern = logicaltestelement("value", MD.Type.BYTE_ARRAY; + logical=unknown, converted=MD.ConvertedType.UTF8) + unsupportedmodern = logicaltestelement("value", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(VARIANT=MD.VariantType(specification_version=Int8(1))), + converted=MD.ConvertedType.UTF8) + physical = [UInt8[0x61], UInt8[0x62]] + @test Parquet._logicalkind(unknownmodern) === nothing + @test Parquet._logicalkind(unsupportedmodern) === nothing + @test Parquet._logicalvalues(unknownmodern, physical) === physical + @test Parquet._logicalvalues(unsupportedmodern, physical) === physical + @test Parquet._physicalvalues(unknownmodern, physical) === physical + + plain = logicaltestelement("value", MD.Type.INT64) + unknownlegacy = logicaltestelement("value", MD.Type.INT64; + converted=MD.ConvertedType.T(999)) + integers = Int64[1, 2] + @test Parquet._logicalvalues(plain, integers) === integers + @test Parquet._logicalvalues(unknownlegacy, integers) === integers + + @test_throws Parquet.FormatError Parquet._logicalkind( + logicaltestelement("bad", MD.Type.INT32; logical=stringtype)) + @test_throws Parquet.FormatError Parquet._logicalkind( + logicaltestelement("bad", MD.Type.INT64; logical=datetype)) + @test_throws Parquet.FormatError Parquet._logicalkind( + logicaltestelement("bad", MD.Type.INT32; converted=MD.ConvertedType.UTF8)) + @test_throws Parquet.FormatError Parquet._logicalkind( + logicaltestelement("bad", MD.Type.INT64; converted=MD.ConvertedType.DATE)) +end + +@testset "DATE scalar conversion" begin + epoch = Date(1970, 1, 1) + @test Parquet._fromparquetdate(Int32(-1)) == Date(1969, 12, 31) + @test Parquet._fromparquetdate(Int32(0)) == epoch + @test Parquet._fromparquetdate(Int32(11016)) == Date(2000, 2, 29) + @test Parquet._toparquetdate(Date(1969, 12, 31)) == Int32(-1) + @test Parquet._toparquetdate(epoch) == Int32(0) + @test Parquet._toparquetdate(Date(2000, 2, 29)) == Int32(11016) + + for days in (typemin(Int32), typemax(Int32)) + @test Parquet._toparquetdate(Parquet._fromparquetdate(days)) == days + end + epochday = Dates.value(epoch) + below = Date(Dates.UTD(epochday + Int64(typemin(Int32)) - 1)) + above = Date(Dates.UTD(epochday + Int64(typemax(Int32)) + 1)) + @test_throws ArgumentError Parquet._toparquetdate(below) + @test_throws ArgumentError Parquet._toparquetdate(above) + @test_throws ArgumentError Parquet._toparquetdate(Date(Dates.UTD(typemin(Int64)))) +end + +@testset "logical vector conversion" begin + datetype = logicaltestelement("date", MD.Type.INT32; + logical=MD.LogicalType(DATE=MD.DateType()), converted=MD.ConvertedType.DATE) + physicaldates = Int32[-1, 0, 11016] + dates = Date[Date(1969, 12, 31), Date(1970, 1, 1), Date(2000, 2, 29)] + @test Parquet._logicalvalues(datetype, physicaldates) == dates + @test Parquet._physicalvalues(datetype, dates) == physicaldates + + optionalphysical = Union{Missing,Int32}[missing, -1, 0, 11016] + optionallogical = Union{Missing,Date}[ + missing, Date(1969, 12, 31), Date(1970, 1, 1), Date(2000, 2, 29)] + decoded = Parquet._logicalvalues(datetype, optionalphysical) + encoded = Parquet._physicalvalues(datetype, optionallogical) + @test decoded isa Vector{Union{Missing,Date}} + @test encoded isa Vector{Union{Missing,Int32}} + @test isequal(decoded, optionallogical) + @test isequal(encoded, optionalphysical) + @test Parquet._logicalvalue(datetype, missing) === missing + @test Parquet._physicalvalue(datetype, missing) === missing + @test_throws Parquet.FormatError Parquet._logicalvalue(datetype, Int64(0)) + @test_throws ArgumentError Parquet._physicalvalue(datetype, Int32(0)) + + stringtype = logicaltestelement("text", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(STRING=MD.StringType()), converted=MD.ConvertedType.UTF8) + first = UInt8[0x61, 0x6c, 0x70, 0x68, 0x61] + physicalstrings = [first, UInt8[0xce, 0xb2]] + strings = Parquet._logicalvalues(stringtype, physicalstrings) + @test strings == ["alpha", "β"] + @test first == UInt8[0x61, 0x6c, 0x70, 0x68, 0x61] + @test Parquet._physicalvalues(stringtype, strings) == physicalstrings + + optionalbytes = Union{Missing,Vector{UInt8}}[UInt8[0xce, 0xba], missing] + optionalstrings = Parquet._logicalvalues(stringtype, optionalbytes) + @test optionalstrings isa Vector{Union{Missing,String}} + @test isequal(optionalstrings, Union{Missing,String}["κ", missing]) + @test isequal(Parquet._physicalvalues(stringtype, optionalstrings), optionalbytes) + + @test_throws Parquet.FormatError Parquet._logicalvalues(stringtype, [UInt8[0xff]]) + invalid = String(copy(UInt8[0xff])) + @test !isvalid(invalid) + @test_throws ArgumentError Parquet._physicalvalues(stringtype, [invalid]) + @test_throws Parquet.FormatError Parquet._logicalvalue(stringtype, Int32(1)) + @test_throws ArgumentError Parquet._physicalvalue(stringtype, UInt8[0x61]) +end + +@testset "logical conversion limits" begin + datetype = logicaltestelement("date", MD.Type.INT32; + logical=MD.LogicalType(DATE=MD.DateType())) + stringtype = logicaltestelement("text", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(STRING=MD.StringType())) + tinycontainer = Parquet.Limits(max_container_elements=1) + tinystring = Parquet.Limits(max_string_bytes=1) + @test_throws Parquet.LimitError Parquet._logicalvalues( + datetype, Int32[0, 1]; limits=tinycontainer) + @test_throws Parquet.LimitError Parquet._physicalvalues( + datetype, Date[Date(1970, 1, 1), Date(1970, 1, 2)]; limits=tinycontainer) + @test_throws Parquet.LimitError Parquet._logicalvalues( + stringtype, [UInt8[0x61, 0x62]]; limits=tinystring) + @test_throws Parquet.LimitError Parquet._physicalvalues( + stringtype, ["ab"]; limits=tinystring) +end diff --git a/test/logical_binary.jl b/test/logical_binary.jl new file mode 100644 index 0000000..56cf226 --- /dev/null +++ b/test/logical_binary.jl @@ -0,0 +1,308 @@ +using UUIDs + +if !isdefined(Parquet, :JSONValue) + Base.include(Parquet, joinpath(@__DIR__, "..", "src", "logical_binary.jl")) +end + +if !@isdefined(MD) + const MD = Parquet.Metadata +end +if !@isdefined(TH) + const TH = Parquet.Thrift +end + +function binarylogicalelement(name, physical; logical=nothing, converted=nothing, + width=nothing, repetition=MD.FieldRepetitionType.OPTIONAL) + return MD.SchemaElement(name=name, type_=physical, + repetition_type=repetition, logicalType=logical, converted_type=converted, + type_length=width === nothing ? nothing : Int32(width)) +end + +function littleuint32(value::UInt32) + return UInt8[UInt8(value & 0xff), UInt8((value >> 8) & 0xff), + UInt8((value >> 16) & 0xff), UInt8(value >> 24)] +end + +@testset "binary logical annotation validation" begin + enum = binarylogicalelement("enum", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(ENUM=MD.EnumType())) + json = binarylogicalelement("json", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(JSON=MD.JsonType())) + bson = binarylogicalelement("bson", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(BSON=MD.BsonType())) + uuid = binarylogicalelement("uuid", MD.Type.FIXED_LEN_BYTE_ARRAY; width=16, + logical=MD.LogicalType(UUID=MD.UUIDType())) + float16 = binarylogicalelement("half", MD.Type.FIXED_LEN_BYTE_ARRAY; width=2, + logical=MD.LogicalType(FLOAT16=MD.Float16Type())) + unknown = binarylogicalelement("null", MD.Type.INT64; + logical=MD.LogicalType(UNKNOWN=MD.NullType())) + interval = binarylogicalelement("interval", MD.Type.FIXED_LEN_BYTE_ARRAY; + width=12, converted=MD.ConvertedType.INTERVAL) + @test Parquet._binarylogicalkind(enum) === :enum + @test Parquet._binarylogicalkind(json) === :json + @test Parquet._binarylogicalkind(bson) === :bson + @test Parquet._binarylogicalkind(uuid) === :uuid + @test Parquet._binarylogicalkind(float16) === :float16 + @test Parquet._binarylogicalkind(unknown) === :unknown + @test Parquet._binarylogicalkind(interval) === :interval + @test Parquet._binarylogicaleltype(enum, Vector{UInt8}) === String + @test Parquet._binarylogicaleltype(json, Vector{UInt8}) === Parquet.JSONValue + @test Parquet._binarylogicaleltype(bson, Vector{UInt8}) === Parquet.BSONValue + @test Parquet._binarylogicaleltype(uuid, Vector{UInt8}) === UUID + @test Parquet._binarylogicaleltype(float16, Vector{UInt8}) === Float16 + @test Parquet._binarylogicaleltype(unknown, Int64) === Missing + @test Parquet._binarylogicaleltype(interval, Vector{UInt8}) === Parquet.Interval + + node = Parquet.SchemaNode(uuid, ["uuid"], Int16(1), Int16(0), Int32(1), + Parquet.SchemaNode[]) + @test Parquet._binarylogicalkind(node) === :uuid + @test Parquet._binarylogicaleltype(node, Vector{UInt8}) === UUID + + legacy = ( + (MD.ConvertedType.ENUM, MD.Type.BYTE_ARRAY, nothing, :enum), + (MD.ConvertedType.JSON, MD.Type.BYTE_ARRAY, nothing, :json), + (MD.ConvertedType.BSON, MD.Type.BYTE_ARRAY, nothing, :bson), + (MD.ConvertedType.INTERVAL, MD.Type.FIXED_LEN_BYTE_ARRAY, 12, :interval), + ) + for (converted, physical, width, expected) in legacy + element = binarylogicalelement("legacy", physical; converted=converted, + width=width) + @test Parquet._binarylogicalkind(element) === expected + end + + modernwins = binarylogicalelement("uuid", MD.Type.FIXED_LEN_BYTE_ARRAY; + width=16, logical=MD.LogicalType(UUID=MD.UUIDType()), + converted=MD.ConvertedType.ENUM) + @test Parquet._binarylogicalkind(modernwins) === :uuid + stringwins = binarylogicalelement("text", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(STRING=MD.StringType()), + converted=MD.ConvertedType.INTERVAL) + @test Parquet._binarylogicalkind(stringwins) === nothing + opaque = MD.LogicalType(unknown_fields=(TH.RawField(100, TH.STRUCT, UInt8[0x00]),)) + opaquemodern = binarylogicalelement("opaque", MD.Type.BYTE_ARRAY; + logical=opaque, converted=MD.ConvertedType.JSON) + @test Parquet._binarylogicalkind(opaquemodern) === nothing + @test Parquet._binarylogicalkind(binarylogicalelement("plain", MD.Type.INT32)) === nothing + + badlogical = ( + binarylogicalelement("enum", MD.Type.INT32; + logical=MD.LogicalType(ENUM=MD.EnumType())), + binarylogicalelement("json", MD.Type.INT32; + logical=MD.LogicalType(JSON=MD.JsonType())), + binarylogicalelement("bson", MD.Type.INT64; + logical=MD.LogicalType(BSON=MD.BsonType())), + binarylogicalelement("uuid", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(UUID=MD.UUIDType())), + binarylogicalelement("uuid", MD.Type.FIXED_LEN_BYTE_ARRAY; width=15, + logical=MD.LogicalType(UUID=MD.UUIDType())), + binarylogicalelement("uuid", MD.Type.FIXED_LEN_BYTE_ARRAY; + logical=MD.LogicalType(UUID=MD.UUIDType())), + binarylogicalelement("half", MD.Type.FLOAT; + logical=MD.LogicalType(FLOAT16=MD.Float16Type())), + binarylogicalelement("half", MD.Type.FIXED_LEN_BYTE_ARRAY; width=4, + logical=MD.LogicalType(FLOAT16=MD.Float16Type())), + binarylogicalelement("interval", MD.Type.BYTE_ARRAY; + converted=MD.ConvertedType.INTERVAL), + binarylogicalelement("interval", MD.Type.FIXED_LEN_BYTE_ARRAY; width=8, + converted=MD.ConvertedType.INTERVAL), + binarylogicalelement("unknown", MD.Type.INT32; + logical=MD.LogicalType(UNKNOWN=MD.NullType()), + repetition=MD.FieldRepetitionType.REQUIRED), + MD.SchemaElement(name="group", num_children=Int32(0), + repetition_type=MD.FieldRepetitionType.OPTIONAL, + logicalType=MD.LogicalType(UNKNOWN=MD.NullType())), + ) + for element in badlogical + @test_throws Parquet.FormatError Parquet._binarylogicalkind(element) + end +end + +@testset "UUID byte order and inverse" begin + element = binarylogicalelement("uuid", MD.Type.FIXED_LEN_BYTE_ARRAY; width=16, + logical=MD.LogicalType(UUID=MD.UUIDType())) + bytes = hex2bytes("00112233445566778899aabbccddeeff") + expected = UUID("00112233-4455-6677-8899-aabbccddeeff") + @test Parquet._binarylogicalvalue(element, bytes) == expected + @test Parquet._binaryphysicalvalue(element, expected) == bytes + values = Union{Missing,Vector{UInt8}}[bytes, missing, + hex2bytes("ffffffffffffffffffffffffffffffff")] + decoded = Parquet._binarylogicalvalues(element, values) + @test decoded isa Vector{Union{Missing,UUID}} + @test isequal(decoded, Union{Missing,UUID}[expected, missing, + UUID("ffffffff-ffff-ffff-ffff-ffffffffffff")]) + encoded = Parquet._binaryphysicalvalues(element, decoded) + @test encoded isa Vector{Union{Missing,Vector{UInt8}}} + @test isequal(encoded, values) + @test Parquet._binarylogicalvalue(element, missing) === missing + @test Parquet._binaryphysicalvalue(element, missing) === missing + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, bytes[1:15]) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, bytes[1:1]) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, "uuid") + @test_throws ArgumentError Parquet._binaryphysicalvalue(element, string(expected)) +end + +@testset "FLOAT16 little-endian inverse" begin + element = binarylogicalelement("half", MD.Type.FIXED_LEN_BYTE_ARRAY; width=2, + logical=MD.LogicalType(FLOAT16=MD.Float16Type())) + patterns = UInt16[0x0000, 0x8000, 0x3c00, 0x7c00, 0xfc00, 0x7e01, 0x0001] + physical = [UInt8[UInt8(bits & 0xff), UInt8(bits >> 8)] for bits in patterns] + decoded = Parquet._binarylogicalvalues(element, physical) + @test reinterpret(UInt16, decoded) == patterns + @test Parquet._binaryphysicalvalues(element, decoded) == physical + optional = Union{Missing,Vector{UInt8}}[physical[2], missing, physical[6]] + optionallogical = Parquet._binarylogicalvalues(element, optional) + @test optionallogical isa Vector{Union{Missing,Float16}} + @test reinterpret(UInt16, collect(skipmissing(optionallogical))) == patterns[[2, 6]] + @test isequal(Parquet._binaryphysicalvalues(element, optionallogical), optional) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, UInt8[0x00]) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue( + element, UInt8[0x00, 0x00, 0x00]) + @test_throws ArgumentError Parquet._binaryphysicalvalue(element, Float32(1)) +end + +@testset "ENUM UTF-8 conversion" begin + modern = binarylogicalelement("enum", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(ENUM=MD.EnumType())) + legacy = binarylogicalelement("enum", MD.Type.BYTE_ARRAY; + converted=MD.ConvertedType.ENUM) + bytes = [collect(codeunits("alpha")), UInt8[0xce, 0xb2]] + for element in (modern, legacy) + @test Parquet._binarylogicalvalues(element, bytes) == ["alpha", "β"] + @test Parquet._binaryphysicalvalues(element, ["alpha", "β"]) == bytes + end + source = copy(bytes[1]) + decoded = Parquet._binarylogicalvalue(modern, source) + source[1] = 0x7a + @test decoded == "alpha" + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(modern, UInt8[0xff]) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(modern, Int32(1)) + @test_throws ArgumentError Parquet._binaryphysicalvalue(modern, UInt8[0x61]) +end + +@testset "tagged JSON and BSON bytes" begin + json = binarylogicalelement("json", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(JSON=MD.JsonType())) + bson = binarylogicalelement("bson", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(BSON=MD.BsonType())) + jsonbytes = collect(codeunits("{\"answer\":42}")) + source = copy(jsonbytes) + taggedjson = Parquet._binarylogicalvalue(json, source) + @test taggedjson isa Parquet.JSONValue + @test taggedjson.bytes == jsonbytes + source[1] = 0x00 + @test taggedjson.bytes == jsonbytes + encodedjson = Parquet._binaryphysicalvalue(json, taggedjson) + encodedjson[1] = 0x00 + @test taggedjson.bytes == jsonbytes + copiedjson = copy(taggedjson) + @test copiedjson == taggedjson && isequal(copiedjson, taggedjson) + @test hash(copiedjson) == hash(taggedjson) + originaljsonhash = hash(copiedjson) + @test_throws CanonicalIndexError setindex!(copiedjson.bytes, 0x00, 1) + @test hash(copiedjson) == originaljsonhash + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(json, + collect(codeunits("syntactically not JSON"))) + @test_throws ArgumentError Parquet.JSONValue( + collect(codeunits("syntactically not JSON"))) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(json, UInt8[0xff]) + @test_throws ArgumentError Parquet.JSONValue(UInt8[0xff]) + @test_throws ArgumentError Parquet._binaryphysicalvalue(json, jsonbytes) + + bsonbytes = UInt8[0x07, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00] + taggedbson = Parquet._binarylogicalvalue(bson, bsonbytes) + @test taggedbson isa Parquet.BSONValue + @test taggedbson.bytes == bsonbytes + bsonbytes[1] = 0x00 + @test taggedbson.bytes[1] == 0x07 + encodedbson = Parquet._binaryphysicalvalue(bson, taggedbson) + encodedbson[1] = 0x00 + @test taggedbson.bytes[1] == 0x07 + copiedbson = copy(taggedbson) + @test copiedbson == taggedbson && hash(copiedbson) == hash(taggedbson) + originalbsonhash = hash(copiedbson) + @test_throws CanonicalIndexError setindex!(copiedbson.bytes, 0x00, 1) + @test hash(copiedbson) == originalbsonhash + @test_throws ArgumentError Parquet._binaryphysicalvalue(bson, taggedbson.bytes) + + optionaljson = Union{Missing,Vector{UInt8}}[jsonbytes, missing] + logicaljson = Parquet._binarylogicalvalues(json, optionaljson) + @test logicaljson isa Vector{Union{Missing,Parquet.JSONValue}} + @test isequal(Parquet._binaryphysicalvalues(json, logicaljson), optionaljson) + optionalbson = Union{Missing,Vector{UInt8}}[collect(taggedbson.bytes), missing] + logicalbson = Parquet._binarylogicalvalues(bson, optionalbson) + @test logicalbson isa Vector{Union{Missing,Parquet.BSONValue}} + @test isequal(Parquet._binaryphysicalvalues(bson, logicalbson), optionalbson) +end + +@testset "legacy INTERVAL little-endian inverse" begin + element = binarylogicalelement("interval", MD.Type.FIXED_LEN_BYTE_ARRAY; + width=12, converted=MD.ConvertedType.INTERVAL) + expected = Parquet.Interval(UInt32(1), UInt32(0x01020304), typemax(UInt32)) + bytes = vcat(littleuint32(expected.months), littleuint32(expected.days), + littleuint32(expected.milliseconds)) + @test Parquet._binarylogicalvalue(element, bytes) == expected + @test Parquet._binaryphysicalvalue(element, expected) == bytes + @test isequal(copy(expected), expected) + @test hash(copy(expected)) == hash(expected) + values = Union{Missing,Vector{UInt8}}[bytes, missing, zeros(UInt8, 12)] + decoded = Parquet._binarylogicalvalues(element, values) + @test decoded isa Vector{Union{Missing,Parquet.Interval}} + @test isequal(decoded, Union{Missing,Parquet.Interval}[ + expected, missing, Parquet.Interval(0, 0, 0)]) + @test isequal(Parquet._binaryphysicalvalues(element, decoded), values) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, bytes[1:11]) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, bytes[1:1]) + @test_throws ArgumentError Parquet._binaryphysicalvalue(element, (1, 2, 3)) + @test_throws ArgumentError Parquet.Interval(-1, 0, 0) + @test_throws ArgumentError Parquet.Interval(0, typemax(UInt32) + UInt64(1), 0) +end + +@testset "UNKNOWN always-null contract" begin + element = binarylogicalelement("null", MD.Type.INT32; + logical=MD.LogicalType(UNKNOWN=MD.NullType())) + @test Parquet._binarylogicalvalue(element, missing) === missing + @test Parquet._binaryphysicalvalue(element, missing) === missing + decoded = Parquet._binarylogicalvalues(element, Missing[missing, missing]) + encoded = Parquet._binaryphysicalvalues(element, decoded) + @test decoded isa Vector{Missing} && all(ismissing, decoded) + @test encoded isa Vector{Missing} && all(ismissing, encoded) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, Int32(0)) + @test_throws Parquet.FormatError Parquet._binarylogicalvalues( + element, Union{Missing,Int32}[missing, 0]) + @test_throws ArgumentError Parquet._binaryphysicalvalue(element, nothing) + @test_throws ArgumentError Parquet._binaryphysicalvalues(element, Any[missing, 0]) +end + +@testset "binary logical limits and fallthrough" begin + json = binarylogicalelement("json", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(JSON=MD.JsonType())) + bson = binarylogicalelement("bson", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(BSON=MD.BsonType())) + enum = binarylogicalelement("enum", MD.Type.BYTE_ARRAY; + logical=MD.LogicalType(ENUM=MD.EnumType())) + tinycontainer = Parquet.Limits(max_container_elements=1) + tinybytes = Parquet.Limits(max_string_bytes=1) + for element in (json, bson, enum) + @test_throws Parquet.LimitError Parquet._binarylogicalvalues( + element, [UInt8[0x61], UInt8[0x62]]; limits=tinycontainer) + end + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + json, UInt8[0x61, 0x62]; limits=tinybytes) + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + bson, UInt8[0x61, 0x62]; limits=tinybytes) + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + enum, UInt8[0x61, 0x62]; limits=tinybytes) + @test_throws Parquet.LimitError Parquet._binaryphysicalvalue( + json, Parquet.JSONValue(UInt8[0x22, 0x61, 0x22]); limits=tinybytes) + @test_throws Parquet.LimitError Parquet._binaryphysicalvalue( + bson, Parquet.BSONValue(UInt8[0x05, 0x00, 0x00, 0x00, 0x00]); + limits=tinybytes) + plain = binarylogicalelement("plain", MD.Type.BYTE_ARRAY) + values = [UInt8[0x61]] + @test Parquet._binarylogicalkind(plain) === nothing + @test Parquet._binarylogicaleltype(plain, Vector{UInt8}) === nothing + @test Parquet._binarylogicalvalue(plain, values[1]) === nothing + @test Parquet._binaryphysicalvalue(plain, values[1]) === nothing + @test Parquet._binarylogicalvalues(plain, values) === nothing + @test Parquet._binaryphysicalvalues(plain, values) === nothing +end diff --git a/test/logical_bson.jl b/test/logical_bson.jl new file mode 100644 index 0000000..d22465d --- /dev/null +++ b/test/logical_bson.jl @@ -0,0 +1,233 @@ +function bsonle32(value::Integer) + raw = reinterpret(UInt32, Int32(value)) + return UInt8[UInt8(raw & 0xff), UInt8((raw >> 8) & 0xff), + UInt8((raw >> 16) & 0xff), UInt8(raw >> 24)] +end + +function bsonle64(value::UInt64) + return UInt8[UInt8((value >> shift) & 0xff) for shift in 0:8:56] +end + +function bsoncstring(value::AbstractString) + bytes = collect(codeunits(value)) + push!(bytes, 0x00) + return bytes +end + +function bsonstring(value::AbstractString) + bytes = collect(codeunits(value)) + return vcat(bsonle32(length(bytes) + 1), bytes, UInt8[0x00]) +end + +function bsonelement(type::Integer, key::AbstractString, payload=UInt8[]) + return vcat(UInt8[UInt8(type)], bsoncstring(key), payload) +end + +function bsondocument(elements...) + body = UInt8[] + for element in elements + append!(body, element) + end + return vcat(bsonle32(length(body) + 5), body, UInt8[0x00]) +end + +function bsonarray(count::Int) + body = UInt8[] + for index in 0:(count - 1) + push!(body, 0x0a) + append!(body, codeunits(string(index))) + push!(body, 0x00) + end + return vcat(bsonle32(length(body) + 5), body, UInt8[0x00]) +end + +function bsonlogicalelement() + return Parquet.Metadata.SchemaElement(name="bson", + type_=Parquet.Metadata.Type.BYTE_ARRAY, + repetition_type=Parquet.Metadata.FieldRepetitionType.OPTIONAL, + logicalType=Parquet.Metadata.LogicalType(BSON=Parquet.Metadata.BsonType())) +end + +@noinline function bsonwritehitslimit(element, value, limits) + try + Parquet._binaryphysicalvalue(element, value; limits=limits) + catch err + err isa Parquet.LimitError || rethrow() + return true + end + return false +end + +function representativebson() + nested = bsondocument(bsonelement(0x0a, "null")) + array = bsondocument(bsonelement(0x10, "0", bsonle32(1)), + bsonelement(0x0a, "1")) + binary = vcat(bsonle32(3), UInt8[0x00, 0x01, 0x02, 0x03]) + oldbinary = vcat(bsonle32(7), UInt8[0x02], bsonle32(3), + UInt8[0x04, 0x05, 0x06]) + regex = vcat(bsoncstring("a.*"), bsoncstring("im")) + scope = bsondocument(bsonelement(0x10, "x", bsonle32(1))) + code = bsonstring("return x") + codewithscope = vcat(bsonle32(4 + length(code) + length(scope)), code, scope) + return bsondocument( + bsonelement(0x01, "double", bsonle64(reinterpret(UInt64, 1.5))), + bsonelement(0x02, "string", bsonstring("λ")), + bsonelement(0x03, "document", nested), + bsonelement(0x04, "array", array), + bsonelement(0x05, "binary", binary), + bsonelement(0x05, "oldbinary", oldbinary), + bsonelement(0x06, "undefined"), + bsonelement(0x07, "objectid", collect(UInt8, 1:12)), + bsonelement(0x08, "boolean", UInt8[0x01]), + bsonelement(0x09, "datetime", zeros(UInt8, 8)), + bsonelement(0x0a, "null"), + bsonelement(0x0b, "regex", regex), + bsonelement(0x0c, "dbpointer", vcat(bsonstring("namespace"), zeros(UInt8, 12))), + bsonelement(0x0d, "javascript", bsonstring("return 1")), + bsonelement(0x0e, "symbol", bsonstring("symbol")), + bsonelement(0x0f, "scope", codewithscope), + bsonelement(0x10, "int32", bsonle32(-1)), + bsonelement(0x11, "timestamp", zeros(UInt8, 8)), + bsonelement(0x12, "int64", bsonle64(typemax(UInt64))), + bsonelement(0x13, "decimal128", zeros(UInt8, 16)), + bsonelement(0x7f, "maxkey"), + bsonelement(0xff, "minkey"), + ) +end + +@testset "BSON 1.1 element structure" begin + bytes = representativebson() + @test Parquet._validatebson(bytes, Parquet.Limits(), + Parquet.FormatError) === nothing + @test Parquet.BSONValue(bytes).bytes == bytes + element = bsonlogicalelement() + tagged = Parquet._binarylogicalvalue(element, bytes) + @test tagged == Parquet.BSONValue(bytes) + @test tagged.bytes isa Base.CodeUnits{UInt8,String} + @test Parquet._binaryphysicalvalue(element, tagged) == bytes + opaque = bsondocument(bsonelement(0x05, "binary", + vcat(bsonle32(2), UInt8[0x00, 0xff, 0xfe]))) + @test collect(Parquet.BSONValue(opaque).bytes) == opaque + for subtype in UInt8[0x09, 0x80, 0xff] + binary = bsondocument(bsonelement(0x05, "binary", + vcat(bsonle32(0), subtype))) + @test Parquet._validatebson(binary, Parquet.Limits(), + Parquet.FormatError) === nothing + end +end + +@testset "BSON malformed documents" begin + empty = bsondocument() + invalid = Vector{UInt8}[] + push!(invalid, UInt8[]) + push!(invalid, UInt8[0x04, 0x00, 0x00, 0x00]) + push!(invalid, vcat(bsonle32(6), UInt8[0x00])) + push!(invalid, vcat(bsonle32(5), UInt8[0x00, 0x00])) + missingterminator = copy(empty) + missingterminator[end] = 0x01 + push!(invalid, missingterminator) + oversized = copy(empty) + oversized[1:4] = bsonle32(6) + push!(invalid, oversized) + push!(invalid, bsondocument(UInt8[0x00])) + push!(invalid, bsondocument(UInt8[0x20, 0x00])) + push!(invalid, bsondocument(UInt8[0x0a, 0x61])) + push!(invalid, bsondocument(UInt8[0x0a, 0xff, 0x00])) + push!(invalid, bsondocument(bsonelement(0x01, "double", zeros(UInt8, 7)))) + push!(invalid, bsondocument(bsonelement(0x08, "bool", UInt8[0x02]))) + push!(invalid, bsondocument(bsonelement(0x02, "string", bsonle32(0)))) + push!(invalid, bsondocument(bsonelement(0x02, "string", + vcat(bsonle32(2), UInt8[0x61, 0x62])))) + push!(invalid, bsondocument(bsonelement(0x02, "string", + vcat(bsonle32(2), UInt8[0xff, 0x00])))) + push!(invalid, bsondocument(bsonelement(0x05, "binary", + vcat(bsonle32(-1), UInt8[0x00])))) + push!(invalid, bsondocument(bsonelement(0x05, "binary", + vcat(bsonle32(0), UInt8[0x0a])))) + push!(invalid, bsondocument(bsonelement(0x05, "binary", + vcat(bsonle32(0), UInt8[0x7f])))) + push!(invalid, bsondocument(bsonelement(0x05, "oldbinary", + vcat(bsonle32(7), UInt8[0x02], bsonle32(2), UInt8[1, 2, 3])))) + push!(invalid, bsondocument(bsonelement(0x0b, "regex", + vcat(bsoncstring("a"), bsoncstring("mi"))))) + push!(invalid, bsondocument(bsonelement(0x0b, "regex", + vcat(bsoncstring("a"), bsoncstring("q"))))) + badarray = bsondocument(bsonelement(0x10, "1", bsonle32(1))) + push!(invalid, bsondocument(bsonelement(0x04, "array", badarray))) + leadingzeroarray = bsondocument(bsonelement(0x0a, "00")) + push!(invalid, bsondocument(bsonelement(0x04, "array", leadingzeroarray))) + nondigitarray = bsondocument(bsonelement(0x0a, "x")) + push!(invalid, bsondocument(bsonelement(0x04, "array", nondigitarray))) + overflowarray = bsondocument(bsonelement(0x0a, repeat("9", 32))) + push!(invalid, bsondocument(bsonelement(0x04, "array", overflowarray))) + badnested = copy(empty) + badnested[1:4] = bsonle32(6) + push!(invalid, bsondocument(bsonelement(0x03, "nested", badnested))) + scope = bsondocument() + code = bsonstring("x") + badscope = vcat(bsonle32(4 + length(code) + length(scope) + 1), code, scope) + push!(invalid, bsondocument(bsonelement(0x0f, "scope", badscope))) + for bytes in invalid + @test_throws Parquet.FormatError Parquet._validatebson( + bytes, Parquet.Limits(), Parquet.FormatError) + @test_throws ArgumentError Parquet.BSONValue(bytes) + end +end + +@testset "BSON read, write, mutation, and limits" begin + element = bsonlogicalelement() + bytes = representativebson() + value = Parquet._binarylogicalvalue(element, bytes) + originalhash = hash(value) + @test_throws CanonicalIndexError setindex!(value.bytes, 0x01, length(value.bytes)) + @test hash(value) == originalhash + unchecked = Parquet.BSONValue(UInt8[0x05, 0x00, 0x00, 0x00, 0x01], + Val(:validated)) + @test_throws ArgumentError Parquet._binaryphysicalvalue(element, unchecked) + malformed = copy(bytes) + malformed[1:4] = bsonle32(length(bytes) - 1) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue(element, malformed) + + depthtwo = bsondocument(bsonelement(0x03, "child", bsondocument())) + depththree = bsondocument(bsonelement(0x03, "child", + bsondocument(bsonelement(0x03, "child", bsondocument())))) + depthlimit = Parquet.Limits(max_metadata_depth=2) + @test Parquet._binarylogicalvalue(element, depthtwo; limits=depthlimit) isa + Parquet.BSONValue + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + element, depththree; limits=depthlimit) + + oneitem = Parquet.Limits(max_container_elements=1) + @test Parquet._binarylogicalvalue(element, + bsondocument(bsonelement(0x0a, "a")); limits=oneitem) isa Parquet.BSONValue + @test_throws Parquet.LimitError Parquet._binarylogicalvalue(element, + bsondocument(bsonelement(0x0a, "a"), bsonelement(0x0a, "b")); + limits=oneitem) + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + element, bsondocument(); limits=Parquet.Limits(max_string_bytes=4)) + @test_throws Parquet.LimitError Parquet.BSONValue( + bsondocument(); limits=Parquet.Limits(max_string_bytes=4)) + @test Parquet.BSONValue(bsondocument(); + limits=Parquet.Limits(max_string_bytes=5)).bytes == bsondocument() + + largepayload = fill(UInt8(0xff), 256 * 1024) + binary = vcat(bsonle32(length(largepayload)), UInt8[0x00], largepayload) + large = bsondocument(bsonelement(0x0a, "first"), + bsonelement(0x05, "second", binary)) + largevalue = Parquet.BSONValue(large) + earlyelement = Parquet.Limits(max_container_elements=1, + max_string_bytes=length(large)) + @test bsonwritehitslimit(element, largevalue, earlyelement) + allocations = @allocated bsonwritehitslimit(element, largevalue, earlyelement) + @test allocations < length(large) ÷ 4 +end + +@testset "BSON array key allocation" begin + count = 4096 + array = bsondocument(bsonelement(0x04, "array", bsonarray(count))) + limits = Parquet.Limits(max_container_elements=count + 1) + Parquet._validatebson(array, limits, Parquet.FormatError) + allocations = @allocated Parquet._validatebson(array, limits, + Parquet.FormatError) + @test allocations < count +end diff --git a/test/logical_column.jl b/test/logical_column.jl new file mode 100644 index 0000000..a205c4e --- /dev/null +++ b/test/logical_column.jl @@ -0,0 +1,359 @@ +using Dates +using Test + +const LCMD = Parquet.Metadata + +function logicalcolumnmetadata(bytes::Vector{UInt8}) + file = Parquet.File(bytes) + return try + Parquet.Thrift.decode(file.footer.bytes, LCMD.FileMetaData) + finally + close(file) + end +end + +function logicalcolumnvalues(bytes::Vector{UInt8}, name::Symbol=:value) + table = Parquet.Table(bytes) + return try + copy(getproperty(table.columns, name)) + finally + close(table) + end +end + +function logicalcolumnunit(unit::LCMD.TimeUnit) + unit.MILLIS !== nothing && return :millis + unit.MICROS !== nothing && return :micros + unit.NANOS !== nothing && return :nanos + return nothing +end + +function logicalcolumnconverted(kind::Symbol, unit::Symbol, ::Bool) + if kind === :time + unit === :millis && return LCMD.ConvertedType.TIME_MILLIS + unit === :micros && return LCMD.ConvertedType.TIME_MICROS + else + unit === :millis && return LCMD.ConvertedType.TIMESTAMP_MILLIS + unit === :micros && return LCMD.ConvertedType.TIMESTAMP_MICROS + end + return nothing +end + +function logicalcolumntime(unit::Symbol) + unit === :millis && return Time(12, 34, 56, 789) + unit === :micros && return Time(12, 34, 56, 789, 123) + return Time(12, 34, 56, 789, 123, 456) +end + +function logicalcolumntimestamp(unit::Symbol, adjusted::Bool) + unit === :millis && !adjusted && return DateTime(2000, 2, 29, 12, 34, 56, 789) + ticks = unit === :millis ? Int64(951_827_696_789) : + unit === :micros ? Int64(951_827_696_789_123) : + Int64(951_827_696_789_123_456) + return Parquet.Timestamp(ticks, unit, adjusted) +end + +function logicalcolumntimestamptype(unit::Symbol, adjusted::Bool) + unit === :millis && return adjusted ? Parquet.Timestamp{:millis} : DateTime + unit === :micros && return Parquet.Timestamp{:micros} + return Parquet.Timestamp{:nanos} +end + +@testset "LogicalColumn vector contract" begin + values = ["alpha", "beta"] + column = Parquet.LogicalColumn(values, :enum) + @test column isa AbstractVector{String} + @test eltype(column) === String + @test size(column) == (2,) + @test axes(column) == (Base.OneTo(2),) + @test length(column) == 2 + @test parent(column) === values + @test collect(column) == values + column[2] = "gamma" + @test values == ["alpha", "gamma"] + @test column == values + + copied = copy(column) + @test copied isa Parquet.LogicalColumn + @test typeof(copied.spec) === typeof(column.spec) + @test copied == column + @test parent(copied) !== parent(column) + copied[1] = "changed" + @test column[1] == "alpha" + + nulls = Parquet.LogicalColumn(Missing[missing], :enum) + @test eltype(nulls) === Union{Missing,String} + @test isequal(collect(nulls), Union{Missing,String}[missing]) + + time1 = Parquet.LogicalColumn(Time[], :time; unit=:millis, adjusted=false) + time2 = Parquet.LogicalColumn(Time[], :time; unit=:nanos, adjusted=true) + decimal1 = Parquet.LogicalColumn(Parquet.Decimal[], :decimal; + precision=9, scale=0) + decimal2 = Parquet.LogicalColumn(Parquet.Decimal[], :decimal; + precision=20, scale=4) + @test typeof(time1) === typeof(time2) + @test typeof(decimal1) === typeof(decimal2) + typedtime = Parquet.LogicalColumn(Time[Time(0)], :time; unit=:nanos, + adjusted=false) + @test (@inferred typedtime[1]) isa Time + @test (@inferred Parquet._logicalcolumnwriteelement(:value, decimal1.spec, + false, Parquet.Limits())) isa LCMD.SchemaElement +end + +@testset "LogicalColumn ENUM metadata and round trips" begin + expected = Union{Missing,String}["alpha", missing, "omega"] + for pageversion in (:v1, :v2) + column = Parquet.LogicalColumn(copy(expected), :enum) + bytes = Parquet._encodefile((value=column,); pageversion=pageversion, + encoding=:plain) + @test isequal(logicalcolumnvalues(bytes), expected) + element = logicalcolumnmetadata(bytes).schema[2] + @test element.type_ == LCMD.Type.BYTE_ARRAY + @test element.type_length === nothing + @test element.repetition_type == LCMD.FieldRepetitionType.OPTIONAL + @test element.logicalType.ENUM !== nothing + @test element.converted_type == LCMD.ConvertedType.ENUM + end +end + +@testset "LogicalColumn TIME metadata and round trips" begin + for pageversion in (:v1, :v2), unit in (:millis, :micros, :nanos), + adjusted in (false, true) + expected = Time[logicalcolumntime(unit)] + column = Parquet.LogicalColumn(copy(expected), :time; unit=unit, + adjusted=adjusted) + bytes = Parquet._encodefile((value=column,); pageversion=pageversion, + encoding=:plain) + @test logicalcolumnvalues(bytes) == expected + element = logicalcolumnmetadata(bytes).schema[2] + physical = unit === :millis ? LCMD.Type.INT32 : LCMD.Type.INT64 + @test element.type_ == physical + @test element.repetition_type == LCMD.FieldRepetitionType.REQUIRED + @test element.logicalType.TIME.isAdjustedToUTC == adjusted + @test logicalcolumnunit(element.logicalType.TIME.unit) === unit + @test element.converted_type == logicalcolumnconverted(:time, unit, adjusted) + end +end + +@testset "LogicalColumn TIMESTAMP metadata and round trips" begin + for pageversion in (:v1, :v2), unit in (:millis, :micros, :nanos), + adjusted in (false, true) + expected = [logicalcolumntimestamp(unit, adjusted)] + column = Parquet.LogicalColumn(copy(expected), :timestamp; unit=unit, + adjusted=adjusted) + bytes = Parquet._encodefile((value=column,); pageversion=pageversion, + encoding=:plain) + @test logicalcolumnvalues(bytes) == expected + element = logicalcolumnmetadata(bytes).schema[2] + @test element.type_ == LCMD.Type.INT64 + @test element.repetition_type == LCMD.FieldRepetitionType.REQUIRED + @test element.logicalType.TIMESTAMP.isAdjustedToUTC == adjusted + @test logicalcolumnunit(element.logicalType.TIMESTAMP.unit) === unit + @test element.converted_type == logicalcolumnconverted(:timestamp, unit, adjusted) + end +end + +@testset "LogicalColumn empty and all-null parameterized columns" begin + for pageversion in (:v1, :v2), unit in (:millis, :micros, :nanos), + adjusted in (false, true) + T = logicalcolumntimestamptype(unit, adjusted) + emptycolumn = Parquet.LogicalColumn(T[], :timestamp; unit=unit, + adjusted=adjusted) + emptybytes = Parquet._encodefile((value=emptycolumn,); + pageversion=pageversion, encoding=:plain) + emptyvalues = logicalcolumnvalues(emptybytes) + @test isempty(emptyvalues) + @test eltype(emptyvalues) === T + emptyelement = logicalcolumnmetadata(emptybytes).schema[2] + @test emptyelement.repetition_type == LCMD.FieldRepetitionType.REQUIRED + @test emptyelement.logicalType.TIMESTAMP.isAdjustedToUTC == adjusted + @test logicalcolumnunit(emptyelement.logicalType.TIMESTAMP.unit) === unit + + nullcolumn = Parquet.LogicalColumn(Missing[missing, missing], :timestamp; + unit=unit, adjusted=adjusted) + nullbytes = Parquet._encodefile((value=nullcolumn,); + pageversion=pageversion, encoding=:plain) + nullvalues = logicalcolumnvalues(nullbytes) + @test isequal(nullvalues, Union{Missing,T}[missing, missing]) + @test eltype(nullvalues) === Union{Missing,T} + nullelement = logicalcolumnmetadata(nullbytes).schema[2] + @test nullelement.repetition_type == LCMD.FieldRepetitionType.OPTIONAL + @test nullelement.logicalType.TIMESTAMP.isAdjustedToUTC == adjusted + @test logicalcolumnunit(nullelement.logicalType.TIMESTAMP.unit) === unit + end + + for pageversion in (:v1, :v2) + emptycolumn = Parquet.LogicalColumn(Parquet.Decimal[], :decimal; + precision=20, scale=4) + emptybytes = Parquet._encodefile((value=emptycolumn,); + pageversion=pageversion, encoding=:plain) + emptyvalues = logicalcolumnvalues(emptybytes) + @test isempty(emptyvalues) + @test eltype(emptyvalues) === Parquet.Decimal + emptyelement = logicalcolumnmetadata(emptybytes).schema[2] + @test emptyelement.repetition_type == LCMD.FieldRepetitionType.REQUIRED + @test emptyelement.precision == 20 + @test emptyelement.scale == 4 + + nullcolumn = Parquet.LogicalColumn(Missing[missing, missing], :decimal; + precision=20, scale=4) + nullbytes = Parquet._encodefile((value=nullcolumn,); + pageversion=pageversion, encoding=:plain) + nullvalues = logicalcolumnvalues(nullbytes) + @test isequal(nullvalues, + Union{Missing,Parquet.Decimal}[missing, missing]) + @test eltype(nullvalues) === Union{Missing,Parquet.Decimal} + nullelement = logicalcolumnmetadata(nullbytes).schema[2] + @test nullelement.repetition_type == LCMD.FieldRepetitionType.OPTIONAL + @test nullelement.logicalType.DECIMAL.precision == 20 + @test nullelement.logicalType.DECIMAL.scale == 4 + @test nullelement.precision == 20 + @test nullelement.scale == 4 + @test nullelement.converted_type == LCMD.ConvertedType.DECIMAL + end +end + +@testset "LogicalColumn DECIMAL storage boundaries" begin + input = ( + p9=Parquet.LogicalColumn( + Parquet.Decimal[Parquet.Decimal(999_999_999, 2)], :decimal; + precision=9, scale=2), + p10=Parquet.LogicalColumn( + Parquet.Decimal[Parquet.Decimal(9_999_999_999, 2)], :decimal; + precision=10, scale=2), + p18=Parquet.LogicalColumn( + Parquet.Decimal[Parquet.Decimal(big"999999999999999999", 2)], :decimal; + precision=18, scale=2), + p19=Parquet.LogicalColumn( + Parquet.Decimal[Parquet.Decimal(big"9999999999999999999", 2)], :decimal; + precision=19, scale=2), + ) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion, encoding=:plain) + for name in keys(input) + @test logicalcolumnvalues(bytes, name) == collect(getproperty(input, name)) + end + elements = Dict(element.name => element for element in + logicalcolumnmetadata(bytes).schema[2:end]) + @test elements["p9"].type_ == LCMD.Type.INT32 + @test elements["p10"].type_ == LCMD.Type.INT64 + @test elements["p18"].type_ == LCMD.Type.INT64 + @test elements["p19"].type_ == LCMD.Type.FIXED_LEN_BYTE_ARRAY + @test elements["p19"].type_length == 9 + for (name, precision) in zip(("p9", "p10", "p18", "p19"), + Int32[9, 10, 18, 19]) + element = elements[name] + @test element.logicalType.DECIMAL.precision == precision + @test element.logicalType.DECIMAL.scale == 2 + @test element.precision == precision + @test element.scale == 2 + @test element.converted_type == LCMD.ConvertedType.DECIMAL + end + end +end + +@testset "LogicalColumn encoding policies" begin + input = ( + enum=Parquet.LogicalColumn(fill("alpha", 64), :enum), + time=Parquet.LogicalColumn(fill(Time(1), 64), :time; + unit=:micros, adjusted=false), + timestamp=Parquet.LogicalColumn([ + Parquet.Timestamp(index, :micros, false) for index in Int64(1):Int64(64) + ], :timestamp; unit=:micros, adjusted=false), + decimal=Parquet.LogicalColumn(fill( + Parquet.Decimal(big"1234567890123456789", 2), 64), :decimal; + precision=19, scale=2), + ) + policy = (enum=:dictionary, time=:delta_binary_packed, + timestamp=:delta_binary_packed, decimal=:delta_byte_array) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion, + encoding=policy) + for name in keys(input) + @test logicalcolumnvalues(bytes, name) == collect(getproperty(input, name)) + end + chunks = logicalcolumnmetadata(bytes).row_groups[1].columns + @test LCMD.Encoding.RLE_DICTIONARY in chunks[1].meta_data.encodings + @test LCMD.Encoding.DELTA_BINARY_PACKED in chunks[2].meta_data.encodings + @test LCMD.Encoding.DELTA_BINARY_PACKED in chunks[3].meta_data.encodings + @test LCMD.Encoding.DELTA_BYTE_ARRAY in chunks[4].meta_data.encodings + end +end + +@testset "LogicalColumn constructor validation" begin + LC = Parquet.LogicalColumn + @test_throws ArgumentError LC(("not", "a", "vector"), :enum) + @test_throws ArgumentError LC(String[], "enum") + @test_throws ArgumentError LC(String[], :unknown) + @test_throws ArgumentError LC([], :enum) + @test_throws ArgumentError LC(Int[], :enum) + @test_throws ArgumentError LC(String[], :enum; unit=:millis) + @test_throws ArgumentError LC(String[], :enum; adjusted=false) + @test_throws ArgumentError LC(String[], :enum; precision=1) + @test_throws ArgumentError LC(String[], :enum; scale=0) + + @test_throws ArgumentError LC(Time[], :time; adjusted=false) + @test_throws ArgumentError LC(Time[], :time; unit=:seconds, adjusted=false) + @test_throws ArgumentError LC(Time[], :time; unit="millis", adjusted=false) + @test_throws ArgumentError LC(Time[], :time; unit=:millis) + @test_throws ArgumentError LC(Time[], :time; unit=:millis, adjusted=1) + @test_throws ArgumentError LC(DateTime[], :time; unit=:millis, adjusted=false) + @test_throws ArgumentError LC(Time[], :time; unit=:millis, adjusted=false, + precision=9) + + @test_throws ArgumentError LC(DateTime[], :timestamp; adjusted=false) + @test_throws ArgumentError LC(DateTime[], :timestamp; unit=:micros, + adjusted=false) + @test_throws ArgumentError LC(Parquet.Timestamp{:nanos}[], :timestamp; + unit=:micros, adjusted=false) + @test_throws ArgumentError LC(DateTime[], :timestamp; unit=:millis, + adjusted=false, scale=0) + + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; precision=1) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; precision=true, scale=0) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; precision=1, scale=false) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; precision=0, scale=0) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; precision=1, scale=-1) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; precision=1, scale=2) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; + precision=Int64(typemax(Int32)) + 1, scale=0) + @test_throws ArgumentError LC(Int[], :decimal; precision=9, scale=0) + @test_throws ArgumentError LC(Parquet.Decimal[], :decimal; precision=9, + scale=0, adjusted=false) +end + +@testset "LogicalColumn write-time value validation" begin + millis = Parquet.LogicalColumn( + Time[Time(Dates.Nanosecond(1))], :time; unit=:millis, adjusted=false) + micros = Parquet.LogicalColumn( + Time[Time(Dates.Nanosecond(1))], :time; unit=:micros, adjusted=false) + @test_throws ArgumentError Parquet._encodefile((value=millis,)) + @test_throws ArgumentError Parquet._encodefile((value=micros,)) + + timestampvalues = Parquet.Timestamp{:micros}[ + Parquet.Timestamp(0, :micros, true)] + timestamp = Parquet.LogicalColumn(timestampvalues, :timestamp; unit=:micros, + adjusted=true) + timestampvalues[1] = Parquet.Timestamp(0, :micros, false) + @test_throws ArgumentError Parquet._encodefile((value=timestamp,)) + + wrongscale = Parquet.LogicalColumn( + Parquet.Decimal[Parquet.Decimal(1, 3)], :decimal; precision=9, scale=2) + excessdigits = Parquet.LogicalColumn( + Parquet.Decimal[Parquet.Decimal(1_000_000_000, 2)], :decimal; + precision=9, scale=2) + @test_throws ArgumentError Parquet._encodefile((value=wrongscale,)) + @test_throws ArgumentError Parquet._encodefile((value=excessdigits,)) + + invalid = String(UInt8[0xff]) + @test !isvalid(invalid) + badenum = Parquet.LogicalColumn(String[invalid], :enum) + @test_throws ArgumentError Parquet._encodefile((value=badenum,)) + + wideempty = Parquet.LogicalColumn(Parquet.Decimal[], :decimal; + precision=19, scale=2) + limits = Parquet.Limits(max_decimal_bytes=8) + @test_throws Parquet.LimitError Parquet._encodefile((value=wideempty,); + limits=limits) +end diff --git a/test/logical_corpus.jl b/test/logical_corpus.jl new file mode 100644 index 0000000..efe5ab3 --- /dev/null +++ b/test/logical_corpus.jl @@ -0,0 +1,69 @@ +using Test +using UUIDs + +@testset "Apache logical-type corpus" begin + corpus = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + data = joinpath(corpus, "data") + fixture = joinpath(data, "int32_decimal.parquet") + if !isfile(fixture) + @info "parquet-testing corpus not found; skipping logical fixtures" corpus + else + expecteddecimal = Parquet.Decimal[ + Parquet.Decimal(100 * index, 2) for index in 1:24 + ] + for name in ( + "int32_decimal.parquet", + "int64_decimal.parquet", + "byte_array_decimal.parquet", + "fixed_length_decimal.parquet", + "fixed_length_decimal_legacy.parquet") + table = Parquet.Table(joinpath(data, name)) + @test table.columns.value == expecteddecimal + close(table) + end + + nonzero = Parquet.Table(joinpath(data, "float16_nonzeros_and_nans.parquet")) + expectedbits = Union{Missing,UInt16}[ + missing, 0x3c00, 0xc000, 0x7e00, 0x0000, 0xbc00, 0x8000, 0x4000] + actualbits = Union{Missing,UInt16}[ + ismissing(value) ? missing : reinterpret(UInt16, value) + for value in nonzero.columns.x + ] + @test isequal(actualbits, expectedbits) + close(nonzero) + + zeros = Parquet.Table(joinpath(data, "float16_zeros_and_nans.parquet")) + expectedbits = Union{Missing,UInt16}[missing, 0x0000, 0x7e00] + actualbits = Union{Missing,UInt16}[ + ismissing(value) ? missing : reinterpret(UInt16, value) + for value in zeros.columns.x + ] + @test isequal(actualbits, expectedbits) + close(zeros) + + json = Parquet.Table(joinpath(data, "json.parquet")) + expectedjson = Union{Missing,Parquet.JSONValue}[ + Parquet.JSONValue(codeunits("{\"a\":1}")), + Parquet.JSONValue(codeunits("{\"a\":1,\"b\":null}")), + Parquet.JSONValue(codeunits("[1,null,3]")), + missing, + ] + @test isequal(json.columns.json_field, expectedjson) + close(json) + + bson = Parquet.Table(joinpath(data, "bson.parquet")) + expectedbson = Union{Missing,Parquet.BSONValue}[ + Parquet.BSONValue(hex2bytes("0c0000001061000100000000")), + Parquet.BSONValue(hex2bytes("0f000000106100010000000a620000")), + missing, + ] + @test isequal(bson.columns.bson_field, expectedbson) + close(bson) + + unknown = Parquet.Table(joinpath(data, "unknown-logical-type.parquet")) + @test unknown.columns[2] == [ + collect(codeunits("unknown string $index")) for index in 1:3 + ] + close(unknown) + end +end diff --git a/test/logical_decimal.jl b/test/logical_decimal.jl new file mode 100644 index 0000000..e4cd018 --- /dev/null +++ b/test/logical_decimal.jl @@ -0,0 +1,306 @@ +using Test + +const DMD = Parquet.Metadata + +struct DecimalConversionProbe{T} <: AbstractVector{T} + length::Int +end + +function Base.IndexStyle(::Type{<:DecimalConversionProbe}) + return IndexLinear() +end + +function Base.size(values::DecimalConversionProbe) + return (values.length,) +end + +function Base.getindex(::DecimalConversionProbe, ::Int) + throw(ErrorException("DECIMAL conversion probe was indexed")) +end + +function decimalreadallocation(element, values, limits) + caught = Ref{Any}(nothing) + allocated = @allocated caught[] = try + Parquet._logicalvalues(element, values; limits=limits) + nothing + catch err + err + end + return caught[], allocated +end + +function decimalwriteallocation(element, values, limits) + caught = Ref{Any}(nothing) + allocated = @allocated caught[] = try + Parquet._physicalvalues(element, values; limits=limits) + nothing + catch err + err + end + return caught[], allocated +end + +function decimaltestelement(name, physical, precision, scale; width=nothing, + modern=true) + logical = modern ? DMD.LogicalType( + DECIMAL=DMD.DecimalType(precision=Int32(precision), scale=Int32(scale))) : nothing + converted = modern ? DMD.ConvertedType.DECIMAL : DMD.ConvertedType.DECIMAL + return DMD.SchemaElement( + name=String(name), + type_=physical, + type_length=width === nothing ? nothing : Int32(width), + repetition_type=DMD.FieldRepetitionType.OPTIONAL, + converted_type=converted, + precision=modern ? nothing : Int32(precision), + scale=modern ? nothing : Int32(scale), + logicalType=logical, + ) +end + +@testset "DECIMAL metadata validation" begin + for (physical, precision, width) in ( + (DMD.Type.INT32, 9, nothing), + (DMD.Type.INT64, 18, nothing), + (DMD.Type.BYTE_ARRAY, 100, nothing), + (DMD.Type.FIXED_LEN_BYTE_ARRAY, 9, 4)) + element = decimaltestelement("value", physical, precision, 2; width=width) + @test Parquet._decimallogicalkind(element) === :decimal + @test Parquet._decimallogicaleltype(:decimal) === Parquet.Decimal + end + legacy = decimaltestelement("legacy", DMD.Type.INT64, 18, 4; modern=false) + @test Parquet._decimallogicalkind(legacy) === :decimal + @test_throws Parquet.FormatError Parquet._decimallogicalkind( + decimaltestelement("bad", DMD.Type.FLOAT, 5, 2)) + @test_throws Parquet.FormatError Parquet._decimallogicalkind( + decimaltestelement("bad", DMD.Type.INT32, 10, 2)) + @test_throws Parquet.FormatError Parquet._decimallogicalkind( + decimaltestelement("bad", DMD.Type.INT64, 19, 2)) + @test_throws Parquet.FormatError Parquet._decimallogicalkind( + decimaltestelement("bad", DMD.Type.FIXED_LEN_BYTE_ARRAY, 3, 0; width=1)) + @test_throws Parquet.FormatError Parquet._decimallogicalkind( + decimaltestelement("bad", DMD.Type.BYTE_ARRAY, 0, 0)) + @test_throws Parquet.FormatError Parquet._decimallogicalkind( + decimaltestelement("bad", DMD.Type.BYTE_ARRAY, 4, 5)) +end + +@testset "DECIMAL exact values" begin + element = decimaltestelement("value", DMD.Type.BYTE_ARRAY, 20, 4) + cases = ( + (UInt8[0x00], big(0)), + (UInt8[0x7f], big(127)), + (UInt8[0x00, 0x80], big(128)), + (UInt8[0x80], big(-128)), + (UInt8[0xff, 0x7f], big(-129)), + (UInt8[0xff], big(-1)), + ) + for (bytes, unscaled) in cases + value = Parquet._fromparquetdecimal(element, bytes, Parquet.Limits()) + @test value == Parquet.Decimal(unscaled, 4) + @test Parquet._toparquetdecimal(element, value, Parquet.Limits()) == bytes + end + for (unscaled, width) in ( + (big(-129), 2), + (big(-128), 1), + (big(-1), 1), + (big(0), 1), + (big(127), 1), + (big(128), 2), + ) + @test Parquet._twoscomplementwidth(unscaled) == width + end + value = Parquet.Decimal(big"12345678901234567890", 4) + @test copy(value) == value + @test isequal(copy(value), value) + @test hash(copy(value)) == hash(value) + @test sprint(show, value) == "Parquet.Decimal(12345678901234567890, 4)" + @test_throws ArgumentError Parquet.Decimal(1, -1) +end + +@testset "DECIMAL bulk two's-complement conversion" begin + for width in (1, 2, 3, 8, 17, 256, 4096) + bits = 8 * width + minimum = -(big(1) << (bits - 1)) + maximum = (big(1) << (bits - 1)) - 1 + values = (minimum, minimum + 1, big(-1), big(0), big(1), + maximum - 1, maximum) + for value in values + encoded = Parquet._totwoscomplement(value, width) + @test length(encoded) == width + @test Parquet._fromtwoscomplement(encoded) == value + end + @test_throws ArgumentError Parquet._totwoscomplement(minimum - 1, width) + @test_throws ArgumentError Parquet._totwoscomplement(maximum + 1, width) + end + + storage = UInt8[0x00, 0xaa, 0x80, 0xbb] + noncontiguous = @view storage[1:2:4] + @test Parquet._fromtwoscomplement(noncontiguous) == 128 + @test storage == UInt8[0x00, 0xaa, 0x80, 0xbb] + @test_throws Parquet.FormatError Parquet._fromtwoscomplement(UInt8[]) + @test Parquet._totwoscomplement(big(-1), 8) == fill(UInt8(0xff), 8) + @test Parquet._totwoscomplement(big(128), 8) == + UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80] +end + +@testset "DECIMAL physical round trips" begin + int32 = decimaltestelement("int32", DMD.Type.INT32, 9, 2) + int64 = decimaltestelement("int64", DMD.Type.INT64, 18, 6) + fixed = decimaltestelement("fixed", DMD.Type.FIXED_LEN_BYTE_ARRAY, 9, 2; width=4) + for (element, raw, expected) in ( + (int32, Int32(-12345), Parquet.Decimal(-12345, 2)), + (int64, Int64(123456789012345678), + Parquet.Decimal(123456789012345678, 6)), + (fixed, UInt8[0xff, 0xff, 0xcf, 0xc7], Parquet.Decimal(-12345, 2))) + decoded = Parquet._fromparquetdecimal(element, raw, Parquet.Limits()) + @test decoded == expected + @test Parquet._toparquetdecimal(element, decoded, Parquet.Limits()) == raw + end + @test Parquet._toparquetdecimal(fixed, Parquet.Decimal(128, 2), Parquet.Limits()) == + UInt8[0x00, 0x00, 0x00, 0x80] + @test_throws ArgumentError Parquet._toparquetdecimal( + int32, Parquet.Decimal(1, 3), Parquet.Limits()) + @test_throws ArgumentError Parquet._toparquetdecimal( + int32, Parquet.Decimal(1_000_000_000, 2), Parquet.Limits()) + @test_throws Parquet.FormatError Parquet._fromparquetdecimal( + fixed, UInt8[0x01], Parquet.Limits()) + @test_throws Parquet.FormatError Parquet._fromparquetdecimal( + int32, Int64(1), Parquet.Limits()) +end + +@testset "DECIMAL resource bounds" begin + element = decimaltestelement("value", DMD.Type.BYTE_ARRAY, 20, 2) + limits = Parquet.Limits(max_string_bytes=1) + @test_throws Parquet.LimitError Parquet._fromparquetdecimal( + element, UInt8[0x00, 0x01], limits) + @test_throws Parquet.LimitError Parquet._toparquetdecimal( + element, Parquet.Decimal(128, 2), limits) + + @test Parquet.Limits().max_decimal_bytes == 1024 * 1024 + decimal_limits = Parquet.Limits(max_decimal_bytes=4) + decode_error = try + Parquet._fromparquetdecimal(element, UInt8[0x00, 0x00, 0x00, 0x00, 0x01], + decimal_limits) + nothing + catch err + err + end + @test decode_error isa Parquet.LimitError + @test decode_error.resource == :decimal_bytes + @test decode_error.requested == 5 + @test decode_error.maximum == 4 + + large = Parquet.Decimal(big(1) << 31, 2) + encode_error = try + Parquet._toparquetdecimal(element, large, decimal_limits) + nothing + catch err + err + end + @test encode_error isa Parquet.LimitError + @test encode_error.resource == :decimal_bytes + @test encode_error.requested == 5 + @test encode_error.maximum == 4 + + fixed = decimaltestelement("fixed", DMD.Type.FIXED_LEN_BYTE_ARRAY, 11, 2; + width=5) + @test_throws Parquet.LimitError Parquet._toparquetdecimal( + fixed, Parquet.Decimal(1, 2), decimal_limits) + exact = Parquet.Limits(max_decimal_bytes=2, max_string_bytes=2) + @test Parquet._fromparquetdecimal(element, UInt8[0x00, 0x80], exact) == + Parquet.Decimal(128, 2) + @test Parquet._toparquetdecimal(element, Parquet.Decimal(128, 2), exact) == + UInt8[0x00, 0x80] + + oversized = fill(UInt8(0xff), 64 * 1024) + rejection_time = @elapsed try + Parquet._fromparquetdecimal(element, oversized, + Parquet.Limits(max_decimal_bytes=16)) + catch err + err isa Parquet.LimitError || rethrow() + end + @test rejection_time < 1.0 +end + +@testset "DECIMAL fixed-width conversion preflight" begin + element = decimaltestelement("fixed", DMD.Type.FIXED_LEN_BYTE_ARRAY, 19, 2; + width=9) + limits = Parquet.Limits(max_decimal_bytes=8) + readvalues = DecimalConversionProbe{Vector{UInt8}}(1_000_000) + writevalues = DecimalConversionProbe{Parquet.Decimal}(1_000_000) + + decimalreadallocation(element, + DecimalConversionProbe{Vector{UInt8}}(1), limits) + decimalwriteallocation(element, + DecimalConversionProbe{Parquet.Decimal}(1), limits) + GC.gc() + readerror, readallocated = decimalreadallocation(element, readvalues, limits) + writeerror, writeallocated = decimalwriteallocation(element, writevalues, limits) + for error in (readerror, writeerror) + @test error isa Parquet.LimitError + @test error.resource == :decimal_bytes + @test error.requested == 9 + @test error.maximum == 8 + end + @test readallocated < 100_000 + @test writeallocated < 100_000 + + stringlimits = Parquet.Limits(max_decimal_bytes=9, max_string_bytes=8) + readerror, readallocated = decimalreadallocation(element, readvalues, + stringlimits) + writeerror, writeallocated = decimalwriteallocation(element, writevalues, + stringlimits) + for error in (readerror, writeerror) + @test error isa Parquet.LimitError + @test error.resource == :string_bytes + @test error.requested == 9 + @test error.maximum == 8 + end + @test readallocated < 100_000 + @test writeallocated < 100_000 + + exactlimits = Parquet.Limits(max_decimal_bytes=9, max_string_bytes=9) + raw = UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01] + logical = Parquet._logicalvalues(element, Vector{UInt8}[raw]; limits=exactlimits) + @test logical == Parquet.Decimal[Parquet.Decimal(1, 2)] + @test Parquet._physicalvalues(element, logical; limits=exactlimits) == + Vector{UInt8}[raw] +end + +@testset "DECIMAL practical linear scaling" begin + smallwidth = 4 * 1024 + largewidth = 16 * 1024 + smallbytes = fill(UInt8(0x55), smallwidth) + largebytes = fill(UInt8(0x55), largewidth) + smallvalue = (big(1) << (8 * smallwidth - 2)) + 123 + largevalue = (big(1) << (8 * largewidth - 2)) + 123 + Parquet._fromtwoscomplement(smallbytes) + Parquet._totwoscomplement(smallvalue, smallwidth) + GC.gc() + smalldecode = @allocated Parquet._fromtwoscomplement(smallbytes) + largedecode = @allocated Parquet._fromtwoscomplement(largebytes) + smallencode = @allocated Parquet._totwoscomplement(smallvalue, smallwidth) + largeencode = @allocated Parquet._totwoscomplement(largevalue, largewidth) + @test largedecode <= 5 * smalldecode + @test largeencode <= 5 * smallencode + + negative = -largevalue + Parquet._twoscomplementwidth(negative) + Parquet._totwoscomplement(negative, largewidth) + GC.gc() + widthallocated = @allocated Parquet._twoscomplementwidth(negative) + negativeallocated = @allocated Parquet._totwoscomplement( + negative, largewidth) + @test widthallocated < 10_000 + @test negativeallocated <= largeencode + 100_000 + + practicalwidth = 256 * 1024 + practicalbytes = fill(UInt8(0x55), practicalwidth) + practicalexpected = Parquet._fromtwoscomplement(practicalbytes) + practicalvalue = (big(1) << (8 * practicalwidth - 2)) + 123 + elapsed = @elapsed begin + @test Parquet._fromtwoscomplement(practicalbytes) == practicalexpected + @test Parquet._fromtwoscomplement( + Parquet._totwoscomplement(practicalvalue, practicalwidth)) == practicalvalue + end + @test elapsed < 5.0 +end diff --git a/test/logical_json.jl b/test/logical_json.jl new file mode 100644 index 0000000..586aed3 --- /dev/null +++ b/test/logical_json.jl @@ -0,0 +1,143 @@ +function jsonbytes(value::AbstractString) + return collect(codeunits(value)) +end + +function jsonlogicalelement() + return Parquet.Metadata.SchemaElement(name="json", + type_=Parquet.Metadata.Type.BYTE_ARRAY, + repetition_type=Parquet.Metadata.FieldRepetitionType.OPTIONAL, + logicalType=Parquet.Metadata.LogicalType(JSON=Parquet.Metadata.JsonType())) +end + +@noinline function jsonwritehitslimit(element, value, limits) + try + Parquet._binaryphysicalvalue(element, value; limits=limits) + catch err + err isa Parquet.LimitError || rethrow() + return true + end + return false +end + +@testset "JSON complete syntax validation" begin + valid = ( + "null", + "true", + "false", + "0", + "-0", + "1234567890", + "-12.5e+10", + "1E-2", + "\"\"", + "\"escape: \\\" \\\\ \\/ \\b \\f \\n \\r \\t\"", + "\"raw UTF-8: λ\"", + "\"\\u03bb\"", + "\"\\ud834\\udd1e\"", + "\"\\ud800\"", + "\"\\ud800\\u0041\"", + "\"\\udc00\"", + "[]", + "{}", + "[null,true,false,0,\"x\",[],{}]", + "{\"a\":1,\"b\":[2,3],\"c\":{\"d\":false}}", + " \t\r\n { \"duplicate\": 1, \"duplicate\": 2 } \n", + ) + for document in valid + bytes = jsonbytes(document) + @test Parquet._validatejson(bytes, Parquet.Limits(), + Parquet.FormatError) === nothing + @test Parquet.JSONValue(bytes).bytes == bytes + end + + invalid = ( + "", + " ", + "nil", + "tru", + "falsee", + "+1", + "01", + "-01", + "1.", + ".1", + "1e", + "1e+", + "NaN", + "Infinity", + "null null", + "[", + "[1,]", + "[,1]", + "[1 2]", + "{", + "{\"a\"}", + "{\"a\":}", + "{\"a\":1,}", + "{a:1}", + "\"unterminated", + "\"bad \\x escape\"", + "\"bad \\u12xz escape\"", + "\"line\nfeed\"", + ) + for document in invalid + bytes = jsonbytes(document) + @test_throws Parquet.FormatError Parquet._validatejson( + bytes, Parquet.Limits(), Parquet.FormatError) + @test_throws ArgumentError Parquet.JSONValue(bytes) + end + @test_throws Parquet.FormatError Parquet._validatejson( + UInt8[0x22, 0xff, 0x22], Parquet.Limits(), Parquet.FormatError) + @test_throws Parquet.FormatError Parquet._validatejson( + UInt8[0xef, 0xbb, 0xbf, 0x6e, 0x75, 0x6c, 0x6c], + Parquet.Limits(), Parquet.FormatError) +end + +@testset "JSON read, write, mutation, and limits" begin + element = jsonlogicalelement() + bytes = jsonbytes("{\"nested\":[1,{\"ok\":true}]}") + value = Parquet._binarylogicalvalue(element, bytes) + @test value == Parquet.JSONValue(bytes) + @test value.bytes isa Base.CodeUnits{UInt8,String} + @test Parquet._binaryphysicalvalue(element, value) == bytes + originalhash = hash(value) + @test_throws CanonicalIndexError setindex!(value.bytes, 0x5d, length(value.bytes)) + @test hash(value) == originalhash + unchecked = Parquet.JSONValue(jsonbytes("{"), Val(:validated)) + @test_throws ArgumentError Parquet._binaryphysicalvalue(element, unchecked) + @test_throws Parquet.FormatError Parquet._binarylogicalvalue( + element, jsonbytes("{\"broken\":]")) + + depthtwo = jsonbytes("[[0]]") + depththree = jsonbytes("[[[0]]]") + depthlimit = Parquet.Limits(max_metadata_depth=2) + @test Parquet._binarylogicalvalue(element, depthtwo; limits=depthlimit) isa + Parquet.JSONValue + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + element, depththree; limits=depthlimit) + @test_throws Parquet.LimitError Parquet._validatejson( + depththree, depthlimit, ArgumentError) + + oneitem = Parquet.Limits(max_container_elements=1) + @test Parquet._binarylogicalvalue(element, jsonbytes("[1]"); limits=oneitem) isa + Parquet.JSONValue + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + element, jsonbytes("[1,2]"); limits=oneitem) + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + element, jsonbytes("{\"a\":1,\"b\":2}"); limits=oneitem) + @test_throws Parquet.LimitError Parquet._binarylogicalvalue( + element, jsonbytes("null"); limits=Parquet.Limits(max_string_bytes=3)) + @test_throws Parquet.LimitError Parquet.JSONValue( + jsonbytes("null"); limits=Parquet.Limits(max_string_bytes=3)) + @test Parquet.JSONValue(jsonbytes("null"); + limits=Parquet.Limits(max_string_bytes=4)).bytes == jsonbytes("null") + + large = vcat(jsonbytes("[[\""), fill(UInt8(0x61), 256 * 1024), + jsonbytes("\"]]")) + largevalue = Parquet.JSONValue(large) + earlydepth = Parquet.Limits(max_metadata_depth=1, + max_string_bytes=length(large)) + @test jsonwritehitslimit(element, largevalue, earlydepth) + allocations = @allocated jsonwritehitslimit(element, largevalue, earlydepth) + @test allocations < length(large) ÷ 4 +end diff --git a/test/logical_temporal.jl b/test/logical_temporal.jl new file mode 100644 index 0000000..ee5ca18 --- /dev/null +++ b/test/logical_temporal.jl @@ -0,0 +1,362 @@ +using Dates + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end + +function temporaltestunit(unit::Symbol) + unit === :millis && return MD.TimeUnit(MILLIS=MD.MilliSeconds()) + unit === :micros && return MD.TimeUnit(MICROS=MD.MicroSeconds()) + unit === :nanos && return MD.TimeUnit(NANOS=MD.NanoSeconds()) + throw(ArgumentError("unsupported test unit $unit")) +end + +function temporaltestelement(name, physical; logical=nothing, converted=nothing) + return MD.SchemaElement(name=name, type_=physical, + repetition_type=MD.FieldRepetitionType.OPTIONAL, + logicalType=logical, converted_type=converted) +end + +function temporaltesttime(unit::Symbol, adjusted::Bool=true) + physical = unit === :millis ? MD.Type.INT32 : MD.Type.INT64 + annotation = MD.TimeType(isAdjustedToUTC=adjusted, unit=temporaltestunit(unit)) + return temporaltestelement("time", physical; + logical=MD.LogicalType(TIME=annotation)) +end + +function temporaltesttimestamp(unit::Symbol, adjusted::Bool=true) + annotation = MD.TimestampType(isAdjustedToUTC=adjusted, unit=temporaltestunit(unit)) + return temporaltestelement("timestamp", MD.Type.INT64; + logical=MD.LogicalType(TIMESTAMP=annotation)) +end + +function temporaltestinteger(width::Int, signed::Bool) + physical = width == 64 ? MD.Type.INT64 : MD.Type.INT32 + annotation = MD.IntType(bitWidth=Int8(width), isSigned=signed) + return temporaltestelement("integer", physical; + logical=MD.LogicalType(INTEGER=annotation)) +end + +@testset "temporal and integer logical recognition" begin + for unit in (:millis, :micros, :nanos), adjusted in (false, true) + time = temporaltesttime(unit, adjusted) + timestamp = temporaltesttimestamp(unit, adjusted) + timekind = Parquet._temporallogicalkind(time) + timestampkind = Parquet._temporallogicalkind(timestamp) + @test timekind isa Parquet._TimeLogicalKind + @test timestampkind isa Parquet._TimestampLogicalKind + @test timekind.is_adjusted_to_utc == adjusted + @test timestampkind.is_adjusted_to_utc == adjusted + @test Parquet._temporallogicaleltype(time, Int64) === Time + expected = unit === :millis ? (adjusted ? Parquet.Timestamp{:millis} : DateTime) : + unit === :micros ? Parquet.Timestamp{:micros} : Parquet.Timestamp{:nanos} + @test Parquet._temporallogicaleltype(timestamp, Int64) === expected + end + + for width in (8, 16, 32, 64), signed in (false, true) + element = temporaltestinteger(width, signed) + kind = Parquet._temporallogicalkind(element) + signedtypes = (Int8, Int16, Int32, Int64) + unsignedtypes = (UInt8, UInt16, UInt32, UInt64) + expected = (signed ? signedtypes : unsignedtypes)[trailing_zeros(width) - 2] + @test kind isa Parquet._IntegerLogicalKind + @test kind.bitwidth == width + @test kind.signed == signed + @test Parquet._temporallogicaleltype(element, Int64) === expected + end + + node = Parquet.SchemaNode(temporaltesttime(:nanos), ["time"], Int16(1), + Int16(0), Int32(1), Parquet.SchemaNode[]) + @test Parquet._temporallogicalkind(node) isa Parquet._TimeLogicalKind + @test Parquet._temporallogicaleltype(node, Int64) === Time + + legacy = ( + (MD.ConvertedType.TIME_MILLIS, MD.Type.INT32, Time), + (MD.ConvertedType.TIME_MICROS, MD.Type.INT64, Time), + (MD.ConvertedType.TIMESTAMP_MILLIS, MD.Type.INT64, + Parquet.Timestamp{:millis}), + (MD.ConvertedType.TIMESTAMP_MICROS, MD.Type.INT64, + Parquet.Timestamp{:micros}), + (MD.ConvertedType.INT_8, MD.Type.INT32, Int8), + (MD.ConvertedType.INT_16, MD.Type.INT32, Int16), + (MD.ConvertedType.INT_32, MD.Type.INT32, Int32), + (MD.ConvertedType.INT_64, MD.Type.INT64, Int64), + (MD.ConvertedType.UINT_8, MD.Type.INT32, UInt8), + (MD.ConvertedType.UINT_16, MD.Type.INT32, UInt16), + (MD.ConvertedType.UINT_32, MD.Type.INT32, UInt32), + (MD.ConvertedType.UINT_64, MD.Type.INT64, UInt64), + ) + for (converted, physical, expected) in legacy + element = temporaltestelement("legacy", physical; converted=converted) + @test Parquet._temporallogicalkind(element) !== nothing + @test Parquet._temporallogicaleltype(element, Int64) === expected + end +end + +@testset "modern annotation precedence and unsupported units" begin + modern = temporaltestelement("value", MD.Type.INT64; + logical=MD.LogicalType(TIMESTAMP=MD.TimestampType( + isAdjustedToUTC=false, unit=temporaltestunit(:nanos))), + converted=MD.ConvertedType.TIME_MICROS) + kind = Parquet._temporallogicalkind(modern) + @test kind isa Parquet._TimestampLogicalKind + @test kind.unit == Parquet._TEMPORAL_NANOS + @test !kind.is_adjusted_to_utc + + stringlogical = MD.LogicalType(STRING=MD.StringType()) + modernstring = temporaltestelement("value", MD.Type.BYTE_ARRAY; + logical=stringlogical, converted=MD.ConvertedType.TIMESTAMP_MICROS) + @test Parquet._temporallogicalkind(modernstring) === nothing + + unknownunit = MD.TimeUnit( + unknown_fields=(TH.RawField(42, TH.STRUCT, UInt8[0x00]),)) + unknowntime = temporaltestelement("value", MD.Type.INT32; + logical=MD.LogicalType(TIME=MD.TimeType( + isAdjustedToUTC=true, unit=unknownunit)), + converted=MD.ConvertedType.TIME_MILLIS) + unknowntimestamp = temporaltestelement("value", MD.Type.INT64; + logical=MD.LogicalType(TIMESTAMP=MD.TimestampType( + isAdjustedToUTC=true, unit=MD.TimeUnit())), + converted=MD.ConvertedType.TIMESTAMP_MICROS) + physical64 = Int64[1, 2] + @test_throws Parquet.UnsupportedFeatureError Parquet._temporallogicalkind( + unknowntime) + @test_throws Parquet.UnsupportedFeatureError Parquet._temporallogicalvalues( + unknowntime, Int32[1, 2]) + @test_throws Parquet.FormatError Parquet._temporallogicalkind(unknowntimestamp) + @test_throws Parquet.FormatError Parquet._temporallogicalvalues( + unknowntimestamp, physical64) + + unknownlogical = MD.LogicalType( + unknown_fields=(TH.RawField(2555, TH.STRUCT, UInt8[0x00]),)) + unknowntype = temporaltestelement("value", MD.Type.INT64; + logical=unknownlogical, converted=MD.ConvertedType.TIMESTAMP_MICROS) + @test Parquet._temporallogicalkind(unknowntype) === nothing + @test Parquet._temporalphysicalvalues(unknowntype, physical64) === physical64 +end + +@testset "temporal and integer schema validation" begin + for unit in (:millis, :micros, :nanos) + wrongtime = unit === :millis ? MD.Type.INT64 : MD.Type.INT32 + time = MD.TimeType(isAdjustedToUTC=true, unit=temporaltestunit(unit)) + @test_throws Parquet.FormatError Parquet._temporallogicalkind( + temporaltestelement("bad", wrongtime; + logical=MD.LogicalType(TIME=time))) + timestamp = MD.TimestampType(isAdjustedToUTC=true, + unit=temporaltestunit(unit)) + @test_throws Parquet.FormatError Parquet._temporallogicalkind( + temporaltestelement("bad", MD.Type.INT32; + logical=MD.LogicalType(TIMESTAMP=timestamp))) + end + + for width in (8, 16, 32, 64) + wrong = width == 64 ? MD.Type.INT32 : MD.Type.INT64 + integer = MD.IntType(bitWidth=Int8(width), isSigned=true) + @test_throws Parquet.FormatError Parquet._temporallogicalkind( + temporaltestelement("bad", wrong; + logical=MD.LogicalType(INTEGER=integer))) + end + for width in (-128, 0, 7, 24, 63, 65) + integer = MD.IntType(bitWidth=Int8(width), isSigned=true) + @test_throws Parquet.FormatError Parquet._temporallogicalkind( + temporaltestelement("bad", MD.Type.INT32; + logical=MD.LogicalType(INTEGER=integer))) + end + + legacybad = ( + (MD.ConvertedType.TIME_MILLIS, MD.Type.INT64), + (MD.ConvertedType.TIME_MICROS, MD.Type.INT32), + (MD.ConvertedType.TIMESTAMP_MILLIS, MD.Type.INT32), + (MD.ConvertedType.TIMESTAMP_MICROS, MD.Type.INT32), + (MD.ConvertedType.INT_8, MD.Type.INT64), + (MD.ConvertedType.INT_64, MD.Type.INT32), + (MD.ConvertedType.UINT_32, MD.Type.INT64), + (MD.ConvertedType.UINT_64, MD.Type.INT32), + ) + for (converted, physical) in legacybad + @test_throws Parquet.FormatError Parquet._temporallogicalkind( + temporaltestelement("bad", physical; converted=converted)) + end +end + +@testset "TIME conversions" begin + cases = ( + (:millis, Int32(45_296_789), Time(12, 34, 56, 789)), + (:micros, Int64(45_296_789_123), Time(12, 34, 56, 789, 123)), + (:nanos, Int64(45_296_789_123_456), Time(12, 34, 56, 789, 123, 456)), + ) + for (unit, physical, expected) in cases, adjusted in (false, true) + element = temporaltesttime(unit, adjusted) + logical = Parquet._temporallogicalvalue(element, physical) + @test logical == expected + @test Parquet._temporalphysicalvalue(element, logical) == physical + end + + boundaries = ( + (:millis, Int32(0), Int32(86_399_999)), + (:micros, Int64(0), Int64(86_399_999_999)), + (:nanos, Int64(0), Int64(86_399_999_999_999)), + ) + for (unit, firstvalue, lastvalue) in boundaries + element = temporaltesttime(unit) + expectedlast = unit === :millis ? Time(23, 59, 59, 999) : + unit === :micros ? Time(23, 59, 59, 999, 999) : + Time(23, 59, 59, 999, 999, 999) + @test Parquet._temporallogicalvalue(element, firstvalue) == Time(0) + @test Parquet._temporallogicalvalue(element, lastvalue) == expectedlast + @test_throws Parquet.FormatError Parquet._temporallogicalvalue( + element, typeof(firstvalue)(-1)) + @test_throws Parquet.FormatError Parquet._temporallogicalvalue( + element, typeof(firstvalue)(lastvalue + 1)) + end + + millis = temporaltesttime(:millis) + micros = temporaltesttime(:micros) + nanos = temporaltesttime(:nanos) + @test_throws Parquet.FormatError Parquet._temporallogicalvalue(millis, Int64(0)) + @test_throws Parquet.FormatError Parquet._temporallogicalvalue(micros, Int32(0)) + @test_throws ArgumentError Parquet._temporalphysicalvalue(millis, Int32(0)) + @test Parquet._temporalphysicalvalue(millis, + Time(Dates.Nanosecond(1_000_000))) == Int32(1) + @test Parquet._temporalphysicalvalue(micros, + Time(Dates.Nanosecond(1_000))) == Int64(1) + @test Parquet._temporalphysicalvalue(nanos, + Time(Dates.Nanosecond(1))) == Int64(1) + @test_throws ArgumentError Parquet._temporalphysicalvalue( + millis, Time(Dates.Nanosecond(1))) + @test_throws ArgumentError Parquet._temporalphysicalvalue( + micros, Time(Dates.Nanosecond(1))) + + optionalphysical = Union{Missing,Int64}[missing, 0, 86_399_999_999_999] + optional = Parquet._temporallogicalvalues(nanos, optionalphysical) + @test optional isa Vector{Union{Missing,Time}} + @test isequal(Parquet._temporalphysicalvalues(nanos, optional), optionalphysical) +end + +@testset "TIMESTAMP conversions" begin + millis = temporaltesttimestamp(:millis, false) + values = ( + (Int64(-1), DateTime(1969, 12, 31, 23, 59, 59, 999)), + (Int64(0), DateTime(1970, 1, 1)), + (Int64(951_827_696_789), DateTime(2000, 2, 29, 12, 34, 56, 789)), + ) + for (physical, expected) in values + @test Parquet._temporallogicalvalue(millis, physical) == expected + @test Parquet._temporalphysicalvalue(millis, expected) == physical + end + + epoch = Dates.value(DateTime(1970, 1, 1)) + upper = typemax(Int64) - epoch + for physical in (typemin(Int64), upper) + logical = Parquet._temporallogicalvalue(millis, physical) + @test Parquet._temporalphysicalvalue(millis, logical) == physical + end + @test_throws Parquet.FormatError Parquet._temporallogicalvalue(millis, upper + 1) + @test_throws ArgumentError Parquet._temporalphysicalvalue( + millis, DateTime(Dates.UTM(typemin(Int64)))) + @test_throws Parquet.FormatError Parquet._temporallogicalvalue(millis, Int32(0)) + @test_throws ArgumentError Parquet._temporalphysicalvalue(millis, Int64(0)) + + for unit in (:millis, :micros, :nanos), adjusted in (false, true), + ticks in (typemin(Int64), Int64(-1), Int64(0), typemax(Int64)) + unit === :millis && !adjusted && continue + element = temporaltesttimestamp(unit, adjusted) + value = Parquet._temporallogicalvalue(element, ticks) + expectedtype = unit === :millis ? Parquet.Timestamp{:millis} : + unit === :micros ? Parquet.Timestamp{:micros} : Parquet.Timestamp{:nanos} + @test value isa expectedtype + @test value.ticks == ticks + @test value.is_adjusted_to_utc == adjusted + @test Parquet._timestampunit(value) === unit + @test Parquet._temporalphysicalvalue(element, value) == ticks + end + + @test isbitstype(Parquet.Timestamp{:millis}) + @test isbitstype(Parquet.Timestamp{:micros}) + @test isbitstype(Parquet.Timestamp{:nanos}) + first = Parquet.Timestamp(Int64(1), :micros, true) + second = Parquet.Timestamp(Int64(1), :micros, true) + @test first == second + @test isequal(first, second) + @test hash(first) == hash(second) + @test sprint(show, first) == "Timestamp(1, :micros, true)" + @test sprint(show, Parquet.Timestamp(Int64(2), :millis, true)) == + "Timestamp(2, :millis, true)" + @test_throws ArgumentError Parquet.Timestamp(Int64(1), :seconds, true) + # millis + adjusted=false is represented as Dates.DateTime, so the type rejects it + # at construction with a clear message instead of failing deep in conversion. + @test_throws ArgumentError Parquet.Timestamp(Int64(0), :millis, false) + @test Parquet.Timestamp(Int64(0), :micros, false) isa Parquet.Timestamp{:micros} + @test Parquet.Timestamp(Int64(0), :nanos, false) isa Parquet.Timestamp{:nanos} + @test_throws ArgumentError Parquet._temporalphysicalvalue( + temporaltesttimestamp(:micros, true), Parquet.Timestamp(1, :nanos, true)) + @test_throws ArgumentError Parquet._temporalphysicalvalue( + temporaltesttimestamp(:micros, true), Parquet.Timestamp(1, :micros, false)) + + optionalphysical = Union{Missing,Int64}[missing, typemin(Int64), typemax(Int64)] + nanos = temporaltesttimestamp(:nanos, true) + optional = Parquet._temporallogicalvalues(nanos, optionalphysical) + @test optional isa Vector{Union{Missing,Parquet.Timestamp{:nanos}}} + @test isequal(Parquet._temporalphysicalvalues(nanos, optional), optionalphysical) +end + +@testset "INTEGER conversions" begin + cases = ( + (8, true, Int32[-128, 0, 127], Int8[-128, 0, 127]), + (16, true, Int32[-32768, 0, 32767], Int16[-32768, 0, 32767]), + (32, true, Int32[typemin(Int32), 0, typemax(Int32)], + Int32[typemin(Int32), 0, typemax(Int32)]), + (64, true, Int64[typemin(Int64), 0, typemax(Int64)], + Int64[typemin(Int64), 0, typemax(Int64)]), + (8, false, Int32[0, 127, 255], UInt8[0, 127, 255]), + (16, false, Int32[0, 32767, 65535], UInt16[0, 32767, 65535]), + (32, false, Int32[0, typemax(Int32), typemin(Int32), -1], + UInt32[0, 0x7fffffff, 0x80000000, 0xffffffff]), + (64, false, Int64[0, typemax(Int64), typemin(Int64), -1], + UInt64[0, 0x7fffffffffffffff, 0x8000000000000000, 0xffffffffffffffff]), + ) + for (width, signed, physical, logical) in cases + element = temporaltestinteger(width, signed) + decoded = Parquet._temporallogicalvalues(element, physical) + @test decoded == logical + @test eltype(decoded) === eltype(logical) + @test Parquet._temporalphysicalvalues(element, logical) == physical + optionalphysical = Union{Missing,eltype(physical)}[missing, first(physical), last(physical)] + optional = Parquet._temporallogicalvalues(element, optionalphysical) + @test eltype(optional) === Union{Missing,eltype(logical)} + @test isequal(Parquet._temporalphysicalvalues(element, optional), optionalphysical) + end + + invalid = ( + (temporaltestinteger(8, true), Int32(-129)), + (temporaltestinteger(8, true), Int32(128)), + (temporaltestinteger(16, true), Int32(-32769)), + (temporaltestinteger(16, true), Int32(32768)), + (temporaltestinteger(8, false), Int32(-1)), + (temporaltestinteger(8, false), Int32(256)), + (temporaltestinteger(16, false), Int32(-1)), + (temporaltestinteger(16, false), Int32(65536)), + ) + for (element, physical) in invalid + @test_throws Parquet.FormatError Parquet._temporallogicalvalue(element, physical) + end + @test_throws Parquet.FormatError Parquet._temporallogicalvalue( + temporaltestinteger(8, true), Int64(1)) + @test_throws ArgumentError Parquet._temporalphysicalvalue( + temporaltestinteger(8, true), Int16(1)) +end + +@testset "temporal conversion limits" begin + limits = Parquet.Limits(max_container_elements=1) + @test_throws Parquet.LimitError Parquet._temporallogicalvalues( + temporaltesttime(:nanos), Int64[0, 1]; limits=limits) + @test_throws Parquet.LimitError Parquet._temporalphysicalvalues( + temporaltesttimestamp(:micros), + [Parquet.Timestamp(0, :micros, true), + Parquet.Timestamp(1, :micros, true)]; limits=limits) + @test_throws Parquet.LimitError Parquet._temporallogicalvalues( + temporaltestinteger(16, false), Int32[0, 1]; limits=limits) +end diff --git a/test/metadata.jl b/test/metadata.jl new file mode 100644 index 0000000..b594f13 --- /dev/null +++ b/test/metadata.jl @@ -0,0 +1,355 @@ +using Random + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end + +const CORPUS_DIR = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + +function corpuspath(parts...) + return joinpath(CORPUS_DIR, parts...) +end + +function metabytes(f) + w = TH.Writer() + f(w) + return w.buffer +end + +function footermetadata(path) + file = Parquet.File(path) + bytes = copy(file.footer.bytes) + encrypted = file.footer.encrypted + close(file) + r = TH.Reader(bytes) + value = TH.decode(r, encrypted ? MD.FileCryptoMetaData : MD.FileMetaData) + return value, bytes, TH.consumed(r) +end + +function pageheaders(path, meta) + bytes = read(path) + headers = MD.PageHeader[] + for rowgroup in meta.row_groups, chunk in rowgroup.columns + md = chunk.meta_data + start = md.data_page_offset + if md.dictionary_page_offset !== nothing && md.dictionary_page_offset > 0 + start = min(start, md.dictionary_page_offset) + end + stop = start + md.total_compressed_size + pos = start + while pos < stop + r = TH.Reader(bytes, pos + 1, length(bytes)) + header = TH.decode(r, MD.PageHeader) + push!(headers, header) + pos += TH.consumed(r) + header.compressed_page_size + end + pos == stop || error("page walk overshoot in $path") + end + return headers +end + +function corpusfiles(parts...) + dir = corpuspath(parts...) + return [joinpath(dir, f) for f in sort(readdir(dir)) if endswith(f, ".parquet") || endswith(f, ".encrypted")] +end + +@testset "generated metadata structs" begin + kv = MD.KeyValue(key="a") + @test kv.value === nothing && kv.unknown_fields == TH.RawField[] + @test kv == MD.KeyValue(key="a") && hash(kv) == hash(MD.KeyValue(key="a")) + @test kv != MD.KeyValue(key="a", value="b") + @test_throws UndefKeywordError MD.KeyValue() + @test MD.ColumnChunk().file_offset == 0 + @test hasfield(MD.SchemaElement, :type_) && !hasfield(MD.SchemaElement, :type) + @test MD.CompressionCodec.LZO.value == 3 && MD.Encoding.BYTE_STREAM_SPLIT.value == 9 + @test MD.Type.INT96.value == 3 && TH.name(MD.Type.T(3)) === :INT96 + @test sprint(show, MD.Encoding.RLE_DICTIONARY) == "Encoding.RLE_DICTIONARY" + @test sprint(show, MD.Encoding.T(10)) == "Encoding.T(10)" + @test MD.LogicalType(STRING=MD.StringType()).STRING !== nothing + @test_throws ArgumentError MD.LogicalType(STRING=MD.StringType(), MAP=MD.MapType()) + @test_throws ArgumentError MD.ColumnOrder(TYPE_ORDER=MD.TypeDefinedOrder(), unknown_fields=(TH.RawField(3, TH.STRUCT, UInt8[0x00]),)) + schema = [MD.SchemaElement(name="root", num_children=Int32(2)), + MD.SchemaElement(name="id", type_=MD.Type.INT64, repetition_type=MD.FieldRepetitionType.REQUIRED, + logicalType=MD.LogicalType(INTEGER=MD.IntType(bitWidth=Int8(64), isSigned=true))), + MD.SchemaElement(name="ts", type_=MD.Type.INT64, repetition_type=MD.FieldRepetitionType.OPTIONAL, + converted_type=MD.ConvertedType.TIMESTAMP_MICROS, + logicalType=MD.LogicalType(TIMESTAMP=MD.TimestampType(isAdjustedToUTC=true, unit=MD.TimeUnit(MICROS=MD.MicroSeconds()))))] + stats = MD.Statistics(null_count=Int64(0), min_value=UInt8[1, 0, 0, 0, 0, 0, 0, 0], + max_value=UInt8[9, 0, 0, 0, 0, 0, 0, 0], is_max_value_exact=true) + column = MD.ColumnMetaData(type_=MD.Type.INT64, encodings=[MD.Encoding.PLAIN, MD.Encoding.RLE], + path_in_schema=["id"], codec=MD.CompressionCodec.ZSTD, num_values=Int64(3), total_uncompressed_size=Int64(40), + total_compressed_size=Int64(30), data_page_offset=Int64(4), statistics=stats, + encoding_stats=[MD.PageEncodingStats(page_type=MD.PageType.DATA_PAGE, encoding=MD.Encoding.PLAIN, count=Int32(1))], + size_statistics=MD.SizeStatistics(definition_level_histogram=Int64[0, 3])) + rowgroup = MD.RowGroup(columns=[MD.ColumnChunk(meta_data=column)], total_byte_size=Int64(40), num_rows=Int64(3), + sorting_columns=[MD.SortingColumn(column_idx=Int32(0), descending=false, nulls_first=true)], ordinal=Int16(0)) + meta = MD.FileMetaData(version=Int32(1), schema=schema, num_rows=Int64(3), row_groups=[rowgroup], + key_value_metadata=[MD.KeyValue(key="k", value="v"), MD.KeyValue(key="novalue")], created_by="Parquet.jl test", + column_orders=[MD.ColumnOrder(TYPE_ORDER=MD.TypeDefinedOrder()), MD.ColumnOrder(IEEE_754_TOTAL_ORDER=MD.IEEE754TotalOrder())]) + bytes = TH.encode(meta) + decoded = TH.decode(bytes, MD.FileMetaData) + @test isequal(decoded, meta) && decoded == meta && hash(decoded) == hash(meta) + @test TH.encode(decoded) == bytes + @test decoded.schema[3].logicalType.TIMESTAMP.unit.MICROS !== nothing + @test decoded.row_groups[1].columns[1].meta_data.statistics.is_max_value_exact === true + @test decoded.row_groups[1].columns[1].meta_data.statistics.is_min_value_exact === nothing + @test decoded.row_groups[1].columns[1].meta_data.statistics.nan_count === nothing + @test decoded.row_groups[1].sorting_columns[1].nulls_first === true + @test decoded.key_value_metadata[2].value === nothing + v2 = MD.DataPageHeaderV2(num_values=Int32(1), num_nulls=Int32(0), num_rows=Int32(1), encoding=MD.Encoding.PLAIN, + definition_levels_byte_length=Int32(0), repetition_levels_byte_length=Int32(0), is_compressed=false) + @test TH.encode(v2)[(end - 1):end] == UInt8[0x12, 0x00] + @test TH.decode(TH.encode(v2), MD.DataPageHeaderV2).is_compressed === false + absent = MD.DataPageHeaderV2(num_values=Int32(1), num_nulls=Int32(0), num_rows=Int32(1), encoding=MD.Encoding.PLAIN, + definition_levels_byte_length=Int32(0), repetition_levels_byte_length=Int32(0)) + @test TH.decode(TH.encode(absent), MD.DataPageHeaderV2).is_compressed === nothing + src = Parquet.source(bytes) + @test TH.decode(Parquet.readrange(src, 0, length(bytes)), MD.FileMetaData) == meta + Parquet.close!(src) +end + +@testset "unknown fields are preserved and re-emitted" begin + bytes = metabytes() do w + lastid = TH.writefieldheader!(w, Int16(0), Int16(1), TH.BINARY) + TH.writestring!(w, "k") + lastid = TH.writefieldheader!(w, lastid, Int16(2), TH.BINARY) + TH.writestring!(w, "v") + lastid = TH.writefieldheader!(w, lastid, Int16(3), TH.I32) + TH.writei32!(w, Int32(42)) + lastid = TH.writefieldheader!(w, lastid, Int16(4), TH.BOOL_TRUE) + lastid = TH.writefieldheader!(w, lastid, Int16(5), TH.LIST) + TH.writelist!(w, [MD.KeyValue(key="nested")]) + lastid = TH.writefieldheader!(w, lastid, Int16(32767), TH.BINARY) + TH.writebinary!(w, UInt8[0xde, 0xad]) + TH.writestop!(w) + end + kv = TH.decode(bytes, MD.KeyValue) + @test kv.key == "k" && kv.value == "v" + @test [f.id for f in kv.unknown_fields] == Int16[3, 4, 5, 32767] + @test [f.type for f in kv.unknown_fields] == UInt8[TH.I32, TH.BOOL_TRUE, TH.LIST, TH.BINARY] + @test TH.payload(kv.unknown_fields[4]) == UInt8[0x02, 0xde, 0xad] + @test kv.unknown_fields[4].bytes[1:4] == UInt8[0x08, 0xfe, 0xff, 0x03] + @test kv.unknown_fields[4].headerlength == 4 && kv.unknown_fields[4].previd == 5 + @test TH.encode(kv) == bytes + leading = metabytes() do w + lastid = TH.writefieldheader!(w, Int16(0), Int16(9), TH.I64) + TH.writei64!(w, Int64(-1)) + lastid = TH.writefieldheader!(w, lastid, Int16(1), TH.BINARY) + TH.writestring!(w, "k") + TH.writestop!(w) + end + lead = TH.decode(leading, MD.KeyValue) + @test lead.unknown_fields[1].previd == 0 && lead.key == "k" + @test TH.encode(lead) == leading + edited = MD.KeyValue(key="kk", value=nothing, unknown_fields=kv.unknown_fields) + reread = TH.decode(TH.encode(edited), MD.KeyValue) + @test reread.key == "kk" && reread.value === nothing + @test reread.unknown_fields == kv.unknown_fields + manual = MD.KeyValue(key="k", unknown_fields=(TH.RawField(32767, TH.BINARY, UInt8[0x01, 0xaa]),)) + @test TH.encode(manual) == UInt8[0x18, 0x01, 0x6b, 0x08, 0xfe, 0xff, 0x03, 0x01, 0xaa, 0x00] + @test TH.decode(TH.encode(manual), MD.KeyValue).unknown_fields == manual.unknown_fields + mismatched = metabytes() do w + lastid = TH.writefieldheader!(w, Int16(0), Int16(1), TH.I32) + TH.writei32!(w, Int32(7)) + lastid = TH.writefieldheader!(w, lastid, Int16(2), TH.I64) + TH.writei64!(w, Int64(0)) + TH.writestop!(w) + end + chunk = TH.decode(mismatched, MD.ColumnChunk) + @test chunk.file_path === nothing && chunk.unknown_fields[1].id == 1 && chunk.unknown_fields[1].type == TH.I32 + @test TH.encode(chunk) == mismatched + wrongkey = metabytes() do w + TH.writefieldheader!(w, Int16(0), Int16(1), TH.I32) + TH.writei32!(w, Int32(1)) + TH.writestop!(w) + end + @test_throws Parquet.FormatError TH.decode(wrongkey, MD.KeyValue) + listbytes = metabytes() do w + TH.writefieldheader!(w, Int16(0), Int16(2), TH.LIST) + TH.writelist!(w, ["x"]) + TH.writestop!(w) + end + sizes = TH.decode(listbytes, MD.SizeStatistics) + @test sizes.repetition_level_histogram === nothing && sizes.unknown_fields[1].id == 2 && sizes.unknown_fields[1].type == TH.LIST + @test TH.encode(sizes) == listbytes + order = TH.decode(UInt8[0x3c, 0x00, 0x00], MD.ColumnOrder) + @test order.TYPE_ORDER === nothing && order.IEEE_754_TOTAL_ORDER === nothing && order.unknown_fields[1].id == 3 + @test TH.encode(order) == UInt8[0x3c, 0x00, 0x00] + @test_throws Parquet.FormatError TH.decode(UInt8[0x1c, 0x00, 0x2c, 0x00, 0x00], MD.ColumnOrder) + @test_throws Parquet.FormatError TH.decode(UInt8[0x1c, 0x00, 0x1c, 0x00, 0x00], MD.LogicalType) +end + +@testset "many unknown fields decode to a Vector, not a per-length tuple type" begin + # Regression: unknown_fields was Tuple{Vararg{RawField}}, so a footer with + # thousands of unknown fields minted a fresh NTuple{N} type whose recursive + # ==/isequal/hash forced seconds of compilation per distinct N. + bytes = metabytes() do w + lastid = TH.writefieldheader!(w, Int16(0), Int16(1), TH.BINARY) + TH.writestring!(w, "k") + for id in Int16(100):Int16(1599) + lastid = TH.writefieldheader!(w, lastid, id, TH.I32) + TH.writei32!(w, Int32(id)) + end + TH.writestop!(w) + end + kv = TH.decode(bytes, MD.KeyValue) + @test kv.unknown_fields isa Vector{TH.RawField} + @test length(kv.unknown_fields) == 1500 + other = TH.decode(bytes, MD.KeyValue) + @test kv == other && isequal(kv, other) && hash(kv) == hash(other) + @test TH.encode(kv) == bytes +end + +@testset "empty unknown-field vectors are budgeted" begin + bytes = TH.encode(MD.StringType()) + vectorcharge = Parquet._materializedarraybytes(TH.RawField, 0) + exact = Parquet._materializedsum(Parquet._MATERIALIZED_OBJECT_BYTES, + vectorcharge) + @test_throws Parquet.LimitError TH.decode(bytes, MD.StringType; + limits=Parquet.Limits(max_materialized_bytes=exact - 1)) + limits = Parquet.Limits(max_materialized_bytes=exact) + budget = Parquet._LiveByteBudget(limits) + value = TH.decode(bytes, MD.StringType; limits=limits, budget=budget) + @test isempty(value.unknown_fields) + @test Parquet._budgetused(budget) == exact +end + +@testset "parquet-testing corpus footers" begin + if !isdir(corpuspath("data")) + @warn "parquet-testing corpus not found; skipping corpus tests" CORPUS_DIR + else + files = corpusfiles("data") + @test length(files) >= 70 + for path in vcat(files, corpusfiles("data", "aes256"), corpusfiles("data", "geospatial"), + corpusfiles("shredded_variant"), corpusfiles("bad_data", "variants")) + meta, bytes, used = footermetadata(path) + @test isequal(TH.decode(TH.encode(meta), typeof(meta)), meta) + @test TH.encode(meta) == bytes[1:used] + end + plain, _, _ = footermetadata(corpuspath("data", "alltypes_plain.parquet")) + @test plain.num_rows == 8 && length(plain.schema) == 12 && length(plain.row_groups) == 1 + @test startswith(plain.created_by, "impala version 1.3.0") + @test plain.schema[1].num_children == 11 && plain.schema[2].type_ == MD.Type.INT32 + headers = pageheaders(corpuspath("data", "alltypes_plain.parquet"), plain) + @test length(headers) == 21 + @test any(h -> h.dictionary_page_header !== nothing, headers) && all(h -> h.crc === nothing, headers) + v2, _, _ = footermetadata(corpuspath("data", "datapage_v2.snappy.parquet")) + v2headers = pageheaders(corpuspath("data", "datapage_v2.snappy.parquet"), v2) + @test length(v2headers) == 8 && any(h -> h.data_page_header_v2 !== nothing, v2headers) + v2header = first(h.data_page_header_v2 for h in v2headers if h.data_page_header_v2 !== nothing) + @test v2header.num_values > 0 && v2header.num_nulls >= 0 && v2header.definition_levels_byte_length >= 0 + crc, _, _ = footermetadata(corpuspath("data", "datapage_v1-corrupt-checksum.parquet")) + crcheaders = pageheaders(corpuspath("data", "datapage_v1-corrupt-checksum.parquet"), crc) + @test length(crcheaders) == 4 && all(h -> h.crc !== nothing, crcheaders) + overflow, _, _ = footermetadata(corpuspath("data", "overflow_i16_page_cnt.parquet")) + @test overflow.num_rows == 40000 + @test length(pageheaders(corpuspath("data", "overflow_i16_page_cnt.parquet"), overflow)) == 40000 + tiny, _, _ = footermetadata(corpuspath("data", "alltypes_tiny_pages.parquet")) + @test length(pageheaders(corpuspath("data", "alltypes_tiny_pages.parquet"), tiny)) == 5805 + tinybytes = read(corpuspath("data", "alltypes_tiny_pages.parquet")) + chunk = tiny.row_groups[1].columns[1] + ci = TH.decode(TH.Reader(tinybytes, chunk.column_index_offset + 1, chunk.column_index_offset + chunk.column_index_length), MD.ColumnIndex) + @test length(ci.null_pages) == 325 && !any(ci.null_pages) && ci.boundary_order == MD.BoundaryOrder.UNORDERED + @test length(ci.null_counts) == 325 && length(ci.min_values) == 325 + oi = TH.decode(TH.Reader(tinybytes, chunk.offset_index_offset + 1, chunk.offset_index_offset + chunk.offset_index_length), MD.OffsetIndex) + @test length(oi.page_locations) == 325 + @test oi.page_locations[1] == MD.PageLocation(offset=Int64(4), compressed_page_size=Int32(109), first_row_index=Int64(0)) + nation, _, _ = footermetadata(corpuspath("data", "nation.dict-malformed.parquet")) + @test nation.num_rows == 25 && nation.created_by == "parquet-mr" && nation.column_orders === nothing + @test isempty(nation.row_groups[1].columns[1].meta_data.encodings) + dictzero, _, _ = footermetadata(corpuspath("data", "dict-page-offset-zero.parquet")) + @test dictzero.row_groups[1].columns[1].meta_data.dictionary_page_offset == 0 + unknownlt, _, _ = footermetadata(corpuspath("data", "unknown-logical-type.parquet")) + lt = unknownlt.schema[3].logicalType + @test lt.STRING === nothing && length(lt.unknown_fields) == 1 + @test lt.unknown_fields[1].id == 2555 && lt.unknown_fields[1].type == TH.STRUCT + @test unknownlt.schema[2].logicalType.STRING !== nothing + int96, _, _ = footermetadata(corpuspath("data", "int96_timestamp_order.parquet")) + @test int96.column_orders[1].TYPE_ORDER === nothing && int96.column_orders[1].unknown_fields[1].id == 3 + @test int96.row_groups[1].columns[1].meta_data.encodings == [MD.Encoding.PLAIN_DICTIONARY, MD.Encoding.BIT_PACKED] + alp, _, _ = footermetadata(corpuspath("data", "alp_extended.zstd.parquet")) + encodings = unique(e for rg in alp.row_groups for c in rg.columns for e in c.meta_data.encodings) + @test MD.Encoding.T(10) in encodings && TH.name(MD.Encoding.T(10)) === nothing + bloom = read(corpuspath("data", "bloom_filter.xxhash.bin")) + r = TH.Reader(bloom) + header = TH.decode(r, MD.BloomFilterHeader) + @test header.numBytes == 1024 && TH.consumed(r) == 16 && length(bloom) == 16 + 1024 + @test header.algorithm.BLOCK !== nothing && header.hash.XXHASH !== nothing && header.compression.UNCOMPRESSED !== nothing + crypto, _, cused = footermetadata(corpuspath("data", "encrypt_columns_and_footer.parquet.encrypted")) + @test crypto isa MD.FileCryptoMetaData && crypto.key_metadata == b"kf" && cused == 20 + @test crypto.encryption_algorithm.AES_GCM_V1 !== nothing && crypto.encryption_algorithm.AES_GCM_CTR_V1 === nothing + ctr, _, _ = footermetadata(corpuspath("data", "encrypt_columns_and_footer_ctr.parquet.encrypted")) + @test ctr.encryption_algorithm.AES_GCM_CTR_V1 !== nothing + plainfooter, pbytes, pused = footermetadata(corpuspath("data", "encrypt_columns_plaintext_footer.parquet.encrypted")) + @test plainfooter isa MD.FileMetaData && plainfooter.footer_signing_key_metadata == b"kf" + @test plainfooter.encryption_algorithm !== nothing && pused == length(pbytes) - 28 + columns = plainfooter.row_groups[1].columns + @test columns[5].crypto_metadata.ENCRYPTION_WITH_COLUMN_KEY.key_metadata == b"kc2" + @test columns[6].crypto_metadata.ENCRYPTION_WITH_COLUMN_KEY.key_metadata == b"kc1" + @test all(c -> c.crypto_metadata === nothing, columns[1:4]) && columns[5].encrypted_column_metadata !== nothing + geo, _, _ = footermetadata(corpuspath("data", "geospatial", "crs-default.parquet")) + geocol = first(c for c in geo.row_groups[1].columns if c.meta_data.geospatial_statistics !== nothing) + @test geocol.meta_data.geospatial_statistics.bbox == MD.BoundingBox(xmin=-111.0, xmax=-104.0, ymin=41.0, ymax=45.0) + @test geocol.meta_data.geospatial_statistics.geospatial_types == Int32[3] + @test any(se -> se.logicalType !== nothing && se.logicalType.GEOMETRY !== nothing && se.logicalType.GEOMETRY.crs === nothing, geo.schema) + geog, _, _ = footermetadata(corpuspath("data", "geospatial", "geography-points.parquet")) + @test any(se -> se.logicalType !== nothing && se.logicalType.GEOGRAPHY !== nothing && + se.logicalType.GEOGRAPHY.algorithm == MD.EdgeInterpolationAlgorithm.SPHERICAL, geog.schema) + variant, _, _ = footermetadata(corpuspath("shredded_variant", "case-001.parquet")) + @test any(se -> se.logicalType !== nothing && se.logicalType.VARIANT !== nothing && + se.logicalType.VARIANT.specification_version == 1, variant.schema) + @test_throws Parquet.FormatError footermetadata(corpuspath("bad_data", "ARROW-GH-41317.parquet")) + corrupt, _, _ = footermetadata(corpuspath("bad_data", "PARQUET-1481.parquet")) + @test corrupt.schema[2].type_ == MD.Type.T(-7) && TH.name(corrupt.schema[2].type_) === nothing + for name in ("ARROW-GH-41321.parquet", "ARROW-GH-43605.parquet", "ARROW-GH-45185.parquet", + "ARROW-RS-GH-6229-DICTHEADER.parquet", "ARROW-RS-GH-6229-LEVELS.parquet") + meta, bytes, used = footermetadata(corpuspath("bad_data", name)) + @test TH.encode(meta) == bytes[1:used] + end + plainfile = Parquet.File(corpuspath("data", "alltypes_plain.parquet")) + @test TH.decode(plainfile.footer.bytes, MD.FileMetaData) == plain + @test_throws Parquet.LimitError TH.decode(plainfile.footer.bytes, MD.FileMetaData; limits=Parquet.Limits(max_metadata_depth=3)) + @test_throws Parquet.LimitError TH.decode(plainfile.footer.bytes, MD.FileMetaData; limits=Parquet.Limits(max_container_elements=5)) + @test_throws Parquet.LimitError TH.decode(plainfile.footer.bytes, MD.FileMetaData; limits=Parquet.Limits(max_string_bytes=8)) + close(plainfile) + end +end + +@testset "seeded metadata mutations fail safely" begin + if !isfile(corpuspath("data", "alltypes_plain.parquet")) + @warn "parquet-testing corpus not found; skipping mutation tests" CORPUS_DIR + else + file = Parquet.File(corpuspath("data", "alltypes_plain.parquet")) + seed = copy(file.footer.bytes) + close(file) + limits = Parquet.Limits(max_string_bytes=1024 * 1024, max_container_elements=100_000, + max_metadata_depth=64) + rng = Random.Xoshiro(0x50415251554554) + for _ in 1:5_000 + bytes = copy(seed) + operation = rand(rng, 1:4) + if operation == 1 + index = rand(rng, eachindex(bytes)) + bytes[index] = xor(bytes[index], UInt8(1) << rand(rng, 0:7)) + elseif operation == 2 + resize!(bytes, rand(rng, 0:length(bytes))) + elseif operation == 3 + insert!(bytes, rand(rng, 1:(length(bytes) + 1)), rand(rng, UInt8)) + else + first = rand(rng, eachindex(bytes)) + last = min(length(bytes), first + rand(rng, 0:7)) + rand!(rng, view(bytes, first:last)) + end + try + value = TH.decode(bytes, MD.FileMetaData; limits=limits) + encoded = TH.encode(value) + @test TH.decode(encoded, MD.FileMetaData; limits=limits) == value + catch err + @test err isa Union{Parquet.FormatError, Parquet.LimitError} + end + end + end +end diff --git a/test/nested_reader.jl b/test/nested_reader.jl new file mode 100644 index 0000000..2278436 --- /dev/null +++ b/test/nested_reader.jl @@ -0,0 +1,665 @@ +using Test +import Dates + +if !isdefined(Parquet, :_assemblenested) + Base.include(Parquet, joinpath(@__DIR__, "..", "src", "nested_reader.jl")) +end + +if !@isdefined(NRMD) + const NRMD = Parquet.Metadata +end + +function nrelement(name; physical=nothing, repetition=nothing, + children=nothing, logical=nothing, converted=nothing, width=nothing) + return NRMD.SchemaElement(name=name, type_=physical, + repetition_type=repetition, num_children=children, + logicalType=logical, converted_type=converted, type_length=width) +end + +function nrroot(children) + return nrelement("schema"; children=Int32(children)) +end + +function nrlogical(kind::Symbol) + kind === :list && return NRMD.LogicalType(LIST=NRMD.ListType()) + kind === :map && return NRMD.LogicalType(MAP=NRMD.MapType()) + kind === :string && return NRMD.LogicalType(STRING=NRMD.StringType()) + kind === :date && return NRMD.LogicalType(DATE=NRMD.DateType()) + throw(ArgumentError("unknown nested reader logical type $kind")) +end + +function nrplan(elements; limits=Parquet.Limits()) + schema = Parquet.Schema(NRMD.SchemaElement[elements...]; limits=limits) + return Parquet._nestedplan(schema; limits=limits) +end + +function nrstream(plan, index, repetitions, definitions, values; rows) + leaf = plan.leaves[index].source + return Parquet.LeafStream(UInt64[repetitions...], UInt64[definitions...], + values, leaf.max_repetition_level, leaf.max_definition_level; + expected_rows=rows) +end + +@testset "nested reader structs and logical leaves" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + elements = NRMD.SchemaElement[ + nrroot(3), + nrelement("id"; physical=NRMD.Type.INT32, repetition=required), + nrelement("record"; repetition=optional, children=Int32(1)), + nrelement("name"; physical=NRMD.Type.BYTE_ARRAY, + repetition=required, logical=nrlogical(:string)), + nrelement("day"; physical=NRMD.Type.INT32, repetition=optional, + logical=nrlogical(:date)), + ] + plan = nrplan(elements) + streams = Parquet.LeafStream[ + nrstream(plan, 1, [0, 0, 0], [0, 0, 0], Int32[1, 2, 3]; rows=3), + nrstream(plan, 2, [0, 0, 0], [1, 0, 1], + [UInt8[0x61], UInt8[0x63]]; rows=3), + nrstream(plan, 3, [0, 0, 0], [1, 0, 1], Int32[0, 2]; rows=3), + ] + result = Parquet._assemblenested(plan, streams, 3) + @test result isa Parquet.StructVector + @test length(result) == 3 + @test result[1]["id"] == 1 + @test result[2]["record"] === missing + @test result[3]["record"]["name"] == "c" + @test result[1]["day"] == Dates.Date(1970, 1, 1) + @test result[2]["day"] === missing + @test eltype(result.children[2]) == Union{Missing,Parquet.StructValue} + @test result.children[2].ranks == Int32[0, 1, 1, 2] + @test eltype(result.children[2].children[1]) == String + + empty = Parquet._assemblenested(plan, Parquet.LeafStream[ + nrstream(plan, 1, Int[], Int[], Int32[]; rows=0), + nrstream(plan, 2, Int[], Int[], Vector{UInt8}[]; rows=0), + nrstream(plan, 3, Int[], Int[], Int32[]; rows=0), + ], 0) + @test isempty(empty) + @test length(empty.children) == 3 +end + +@testset "nested reader physical leaf types" begin + required = NRMD.FieldRepetitionType.REQUIRED + elements = NRMD.SchemaElement[ + nrroot(7), + nrelement("flag"; physical=NRMD.Type.BOOLEAN, repetition=required), + nrelement("small"; physical=NRMD.Type.INT32, repetition=required), + nrelement("large"; physical=NRMD.Type.INT64, repetition=required), + nrelement("single"; physical=NRMD.Type.FLOAT, repetition=required), + nrelement("double"; physical=NRMD.Type.DOUBLE, repetition=required), + nrelement("bytes"; physical=NRMD.Type.BYTE_ARRAY, repetition=required), + nrelement("fixed"; physical=NRMD.Type.FIXED_LEN_BYTE_ARRAY, + repetition=required, width=Int32(2)), + ] + plan = nrplan(elements) + streams = Parquet.LeafStream[ + nrstream(plan, 1, [0], [0], Bool[true]; rows=1), + nrstream(plan, 2, [0], [0], Int32[2]; rows=1), + nrstream(plan, 3, [0], [0], Int64[3]; rows=1), + nrstream(plan, 4, [0], [0], Float32[4]; rows=1), + nrstream(plan, 5, [0], [0], Float64[5]; rows=1), + nrstream(plan, 6, [0], [0], [UInt8[0x06]]; rows=1), + nrstream(plan, 7, [0], [0], [UInt8[0x07, 0x08]]; rows=1), + ] + result = Parquet._assemblenested(plan, streams, 1) + @test [result[1][index] for index in 1:5] == Any[true, 2, 3, 4, 5] + @test result[1][6] == UInt8[0x06] + @test result.children[7] isa Parquet.FixedByteArrayVector + @test result[1][7] == UInt8[0x07, 0x08] + + wrongtype = copy(streams) + wrongtype[2] = nrstream(plan, 2, [0], [0], Any["wrong"]; rows=1) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, wrongtype, 1) + wrongwidth = copy(streams) + wrongwidth[7] = nrstream(plan, 7, [0], [0], [UInt8[0x07]]; rows=1) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, wrongwidth, 1) +end + +@testset "nested reader list null, empty, and present states" begin + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("items"; repetition=optional, children=Int32(1), + logical=nrlogical(:list), converted=NRMD.ConvertedType.LIST), + nrelement("list"; repetition=repeated, children=Int32(1)), + nrelement("element"; physical=NRMD.Type.INT32, repetition=optional), + ] + plan = nrplan(elements) + stream = nrstream(plan, 1, [0, 0, 0, 1, 1, 0], + [0, 1, 2, 3, 3, 3], Int32[10, 20, 30]; rows=4) + result = Parquet._assemblenested(plan, [stream], 4) + column = result.children[1] + @test column isa Parquet.ListVector + @test column[1] === missing + @test collect(column[2]) == Int32[] + @test isequal(collect(column[3]), Union{Missing,Int32}[missing, 10, 20]) + @test collect(column[4]) == Int32[30] + @test column.offsets == Int32[0, 0, 0, 3, 4] + @test column.validity == Bool[false, true, true, true] + + continuation = nrstream(plan, 1, [0, 1, 0], [0, 3, 1], Int32[1]; rows=2) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, [continuation], 2) + + altered = nrstream(plan, 1, [0], [3], Int32[1]; rows=1) + push!(altered.values, Int32(2)) + @test_throws Parquet.FormatError Parquet._assemblenested(plan, [altered], 1) +end + +@testset "nested reader list of list" begin + required = NRMD.FieldRepetitionType.REQUIRED + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("outer"; repetition=required, children=Int32(1), + logical=nrlogical(:list)), + nrelement("inner"; repetition=repeated, children=Int32(1), + logical=nrlogical(:list)), + nrelement("element"; physical=NRMD.Type.INT32, + repetition=repeated), + ] + plan = nrplan(elements) + stream = nrstream(plan, 1, [0, 0, 1, 2, 1], [0, 1, 2, 2, 2], + Int32[1, 2, 3]; rows=2) + result = Parquet._assemblenested(plan, [stream], 2) + column = result.children[1] + @test collect(column[1]) == Parquet.ListValue{Int32}[] + @test [collect(item) for item in column[2]] == + [Int32[], Int32[1, 2], Int32[3]] + @test column.values isa Parquet.ListVector +end + +@testset "nested reader unannotated repeated boundary" begin + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("items"; physical=NRMD.Type.INT32, repetition=repeated), + ] + plan = nrplan(elements) + @test plan.root.children[1].annotation === :unannotated_repeated + stream = nrstream(plan, 1, [0, 0, 1, 0], [0, 1, 1, 1], + Int32[1, 2, 3]; rows=3) + result = Parquet._assemblenested(plan, [stream], 3) + @test isempty(result[1]["items"]) + @test collect(result[2]["items"]) == Int32[1, 2] + @test collect(result[3]["items"]) == Int32[3] + + continuation = nrstream(plan, 1, [0, 1, 0], [0, 1, 0], + Int32[1]; rows=2) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, [continuation], 2) +end + +@testset "nested reader list of structs and sibling alignment" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("rows"; repetition=required, children=Int32(1), + logical=nrlogical(:list)), + nrelement("list"; repetition=repeated, children=Int32(2)), + nrelement("x"; physical=NRMD.Type.INT32, repetition=required), + nrelement("x"; physical=NRMD.Type.BYTE_ARRAY, repetition=optional, + logical=nrlogical(:string)), + ] + plan = nrplan(elements) + left = nrstream(plan, 1, [0, 0, 1, 0], [0, 1, 1, 1], + Int32[1, 2, 3]; rows=3) + right = nrstream(plan, 2, [0, 0, 1, 0], [0, 2, 1, 2], + [UInt8[0x61], UInt8[0x63]]; rows=3) + result = Parquet._assemblenested(plan, [left, right], 3) + column = result.children[1] + @test isempty(column[1]) + @test length(column[2]) == 2 + @test column[2][1][1] == 1 + @test column[2][1][2] == "a" + @test column[2][2][2] === missing + @test_throws ArgumentError column[2][1]["x"] + @test column[3][1][1] == 3 + + short = nrstream(plan, 2, [0, 0, 0], [0, 2, 2], + [UInt8[0x61], UInt8[0x63]]; rows=3) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, [left, short], 3) + + disagreement = nrstream(plan, 2, [0, 0, 1, 0], [0, 0, 1, 2], + [UInt8[0x63]]; rows=3) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, [left, disagreement], 3) +end + +@testset "nested reader struct with list" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("record"; repetition=optional, children=Int32(2)), + nrelement("id"; physical=NRMD.Type.INT32, repetition=required), + nrelement("items"; repetition=optional, children=Int32(1), + logical=nrlogical(:list)), + nrelement("list"; repetition=repeated, children=Int32(1)), + nrelement("element"; physical=NRMD.Type.INT32, repetition=optional), + ] + plan = nrplan(elements) + id = nrstream(plan, 1, [0, 0, 0, 0], [0, 1, 1, 1], + Int32[2, 3, 4]; rows=4) + items = nrstream(plan, 2, [0, 0, 0, 0, 1], [0, 1, 2, 3, 4], + Int32[7]; rows=4) + result = Parquet._assemblenested(plan, [id, items], 4) + record = result.children[1] + @test record[1] === missing + @test record[2]["id"] == 2 + @test record[2]["items"] === missing + @test isempty(record[3]["items"]) + @test isequal(collect(record[4]["items"]), + Union{Missing,Int32}[missing, 7]) +end + +@testset "nested reader maps, duplicates, and omitted values" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("lookup"; repetition=optional, children=Int32(1), + logical=nrlogical(:map), converted=NRMD.ConvertedType.MAP), + nrelement("key_value"; repetition=repeated, children=Int32(2)), + nrelement("key"; physical=NRMD.Type.INT32, repetition=required), + nrelement("value"; physical=NRMD.Type.BYTE_ARRAY, + repetition=optional, logical=nrlogical(:string)), + ] + plan = nrplan(elements) + keys = nrstream(plan, 1, [0, 0, 0, 1, 1, 0], [0, 1, 2, 2, 2, 2], + Int32[1, 1, 2, 3]; rows=4) + values = nrstream(plan, 2, [0, 0, 0, 1, 1, 0], [0, 1, 3, 2, 3, 3], + [UInt8[0x61], UInt8[0x62], UInt8[0x63]]; rows=4) + result = Parquet._assemblenested(plan, [keys, values], 4) + lookup = result.children[1] + @test lookup[1] === missing + @test isempty(lookup[2]) + @test isequal(collect(lookup[3]), Pair{Int32,Union{Missing,String}}[ + 1 => "a", 1 => missing, 2 => "b"]) + @test Parquet.maplookup(lookup[3], 1) === missing + @test isequal(Dict(lookup[3]), Dict{Int32,Union{Missing,String}}( + 1 => missing, 2 => "b")) + @test collect(lookup[4]) == Pair{Int32,Union{Missing,String}}[3 => "c"] + + omittedelements = NRMD.SchemaElement[ + nrroot(1), + nrelement("lookup"; repetition=required, children=Int32(1), + logical=nrlogical(:map)), + nrelement("entries"; repetition=repeated, children=Int32(1), + converted=NRMD.ConvertedType.MAP_KEY_VALUE), + nrelement("key"; physical=NRMD.Type.INT32, repetition=required), + ] + omittedplan = nrplan(omittedelements) + omittedkeys = nrstream(omittedplan, 1, [0, 1], [1, 1], + Int32[4, 5]; rows=1) + omitted = Parquet._assemblenested(omittedplan, [omittedkeys], 1) + @test isequal(collect(omitted[1]["lookup"]), Pair{Int32,Missing}[ + 4 => missing, 5 => missing]) +end + +@testset "nested reader map of struct and optional key rejection" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("lookup"; repetition=required, children=Int32(1), + logical=nrlogical(:map)), + nrelement("entries"; repetition=repeated, children=Int32(2)), + nrelement("key"; physical=NRMD.Type.INT32, repetition=required), + nrelement("value"; repetition=optional, children=Int32(2)), + nrelement("left"; physical=NRMD.Type.INT32, repetition=required), + nrelement("right"; physical=NRMD.Type.BYTE_ARRAY, + repetition=optional, logical=nrlogical(:string)), + ] + plan = nrplan(elements) + keys = nrstream(plan, 1, [0, 0, 1], [0, 1, 1], Int32[1, 2]; rows=2) + left = nrstream(plan, 2, [0, 0, 1], [0, 2, 1], Int32[10]; rows=2) + right = nrstream(plan, 3, [0, 0, 1], [0, 3, 1], + [UInt8[0x78]]; rows=2) + result = Parquet._assemblenested(plan, [keys, left, right], 2) + lookup = result[2]["lookup"] + @test lookup[1].first == 1 + @test lookup[1].second["left"] == 10 + @test lookup[1].second["right"] == "x" + @test isequal(lookup[2], 2 => missing) + + optionalkeyelements = NRMD.SchemaElement[ + nrroot(1), + nrelement("lookup"; repetition=required, children=Int32(1), + logical=nrlogical(:map)), + nrelement("entries"; repetition=repeated, children=Int32(2)), + nrelement("key"; physical=NRMD.Type.INT32, repetition=optional), + nrelement("value"; physical=NRMD.Type.INT32, repetition=optional), + ] + optionalkeyplan = nrplan(optionalkeyelements) + nullkey = nrstream(optionalkeyplan, 1, [0], [1], Int32[]; rows=1) + nullvalue = nrstream(optionalkeyplan, 2, [0], [1], Int32[]; rows=1) + @test_throws Parquet.FormatError Parquet._assemblenested( + optionalkeyplan, [nullkey, nullvalue], 1) + + presentkey = nrstream(optionalkeyplan, 1, [0], [2], Int32[7]; rows=1) + present = Parquet._assemblenested(optionalkeyplan, + [presentkey, nullvalue], 1) + @test !(Missing <: eltype(present.children[1].keys)) + @test isequal(collect(present[1]["lookup"]), + Pair{Int32,Union{Missing,Int32}}[7 => missing]) +end + +@testset "nested reader malformed alignment and resource order" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("record"; repetition=optional, children=Int32(2)), + nrelement("left"; physical=NRMD.Type.INT32, repetition=required), + nrelement("right"; physical=NRMD.Type.INT32, repetition=required), + ] + plan = nrplan(elements) + left = nrstream(plan, 1, [0], [0], Int32[]; rows=1) + right = nrstream(plan, 2, [0], [1], Int32[1]; rows=1) + tiny = Parquet.Limits(max_materialized_bytes=1) + @test_throws Parquet.LimitError Parquet._assemblenested( + plan, [left, right], 1; limits=tiny, + budget=Parquet._LiveByteBudget(tiny)) + + validleft = nrstream(plan, 1, [0], [1], Int32[1]; rows=1) + passbytes = Parquet._nestedreadpassbytes(plan) + passlimits = Parquet.Limits(max_materialized_bytes=passbytes) + invalidbudget = Parquet._LiveByteBudget(passlimits) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, [left, right], 1; limits=passlimits, + budget=invalidbudget) + @test Parquet._budgetused(invalidbudget) == 0 + finalbudget = Parquet._LiveByteBudget(passlimits) + @test_throws Parquet.LimitError Parquet._assemblenested( + plan, [validleft, right], 1; limits=passlimits, + budget=finalbudget) + @test Parquet._budgetused(finalbudget) == 0 + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, [validleft], 1) + @test_throws Parquet.FormatError Parquet._assemblenested( + plan, [validleft, right], 2) + + budget = Parquet._LiveByteBudget(Parquet.Limits()) + output = Parquet._assemblenested(plan, [validleft, right], 1; + budget=budget) + @test output[1]["record"]["left"] == 1 + @test Parquet._budgetused(budget) > 0 +end + +@testset "nested reader required leafless structs" begin + required = NRMD.FieldRepetitionType.REQUIRED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("empty"; repetition=required, children=Int32(0)), + ] + plan = nrplan(elements) + result = Parquet._assemblenested(plan, Parquet.LeafStream[], 3) + @test length(result) == 3 + @test length(result[1]["empty"]) == 0 + @test result.children[1].rows == 3 +end + +@testset "nested reader list of maps, Bool keys, and zero rows" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("outer"; repetition=optional, children=Int32(1), + logical=nrlogical(:list)), + nrelement("list"; repetition=repeated, children=Int32(1)), + nrelement("element"; repetition=required, children=Int32(1), + logical=nrlogical(:map)), + nrelement("key_value"; repetition=repeated, children=Int32(2)), + nrelement("key"; physical=NRMD.Type.BOOLEAN, repetition=required), + nrelement("value"; physical=NRMD.Type.INT32, repetition=optional), + ] + plan = nrplan(elements) + repetitions = [0, 0, 0, 1, 2, 0] + keys = nrstream(plan, 1, repetitions, [0, 1, 2, 3, 3, 3], + Bool[true, false, true]; rows=4) + values = nrstream(plan, 2, repetitions, [0, 1, 2, 4, 3, 4], + Int32[10, 20]; rows=4) + result = Parquet._assemblenested(plan, [keys, values], 4) + outer = result.children[1] + @test outer[1] === missing + @test isempty(outer[2]) + @test isempty(outer[3][1]) + @test isequal(collect(outer[3][2]), Pair{Bool,Union{Missing,Int32}}[ + true => 10, false => missing]) + @test collect(outer[4][1]) == Pair{Bool,Union{Missing,Int32}}[true => 20] + @test outer.values.keys isa Vector{Bool} + @test Parquet.maplookup(outer[3][2], false) === missing + @test isequal(Dict(outer[3][2]), + Dict{Bool,Union{Missing,Int32}}(true => 10, false => missing)) + + emptykeys = nrstream(plan, 1, Int[], Int[], Bool[]; rows=0) + emptyvalues = nrstream(plan, 2, Int[], Int[], Int32[]; rows=0) + empty = Parquet._assemblenested(plan, [emptykeys, emptyvalues], 0) + @test isempty(empty) + @test isempty(empty.children[1]) + @test isempty(empty.children[1].values) + @test isempty(empty.children[1].values.keys) +end + +@testset "nested reader complex map keys" begin + required = NRMD.FieldRepetitionType.REQUIRED + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("lookup"; repetition=required, children=Int32(1), + logical=nrlogical(:map)), + nrelement("entries"; repetition=repeated, children=Int32(2)), + nrelement("key"; repetition=required, children=Int32(2)), + nrelement("id"; physical=NRMD.Type.INT32, repetition=required), + nrelement("items"; repetition=required, children=Int32(1), + logical=nrlogical(:list)), + nrelement("list"; repetition=repeated, children=Int32(1)), + nrelement("element"; physical=NRMD.Type.INT32, repetition=required), + nrelement("value"; physical=NRMD.Type.INT32, repetition=required), + ] + plan = nrplan(elements) + ids = nrstream(plan, 1, [0, 1], [1, 1], Int32[1, 2]; rows=1) + items = nrstream(plan, 2, [0, 2, 1], [2, 2, 1], + Int32[10, 11]; rows=1) + values = nrstream(plan, 3, [0, 1], [1, 1], Int32[100, 200]; rows=1) + result = Parquet._assemblenested(plan, [ids, items, values], 1) + lookup = result[1]["lookup"] + @test length(lookup) == 2 + @test lookup[1].first["id"] == 1 + @test collect(lookup[1].first["items"]) == Int32[10, 11] + @test isempty(lookup[2].first["items"]) + dictionary = Dict(lookup) + fresh = Parquet.StructVector(["id", "items"], + (Int32[1], Parquet.ListVector(Int32[0, 2], Int32[10, 11])))[1] + @test dictionary[fresh] == 100 + lookup.keys.children[1][1] = Int32(9) + lookup.keys.children[2].values[1] = Int32(99) + @test dictionary[fresh] == 100 + @test !haskey(dictionary, lookup[1].first) +end + +@testset "nested reader special-name legacy lists" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(2), + nrelement("items"; repetition=required, children=Int32(1), + converted=NRMD.ConvertedType.LIST), + nrelement("array"; repetition=repeated, children=Int32(1)), + nrelement("value"; physical=NRMD.Type.INT32, repetition=required), + nrelement("pairs"; repetition=required, children=Int32(1), + converted=NRMD.ConvertedType.LIST), + nrelement("pairs_tuple"; repetition=repeated, children=Int32(1)), + nrelement("member"; physical=NRMD.Type.INT32, repetition=optional), + ] + plan = nrplan(elements) + @test plan.root.children[1].rule == UInt8(4) + @test plan.root.children[2].rule == UInt8(5) + items = nrstream(plan, 1, [0, 0, 1], [0, 1, 1], + Int32[10, 20]; rows=2) + pairs = nrstream(plan, 2, [0, 0], [1, 2], Int32[7]; rows=2) + result = Parquet._assemblenested(plan, [items, pairs], 2) + @test isempty(result[1]["items"]) + @test length(result[2]["items"]) == 2 + @test result[2]["items"][1]["value"] == 10 + @test result[2]["items"][2]["value"] == 20 + @test result[1]["pairs"][1]["member"] === missing + @test result[2]["pairs"][1]["member"] == 7 +end + +@testset "nested reader repetition projection and all-null required struct" begin + required = NRMD.FieldRepetitionType.REQUIRED + optional = NRMD.FieldRepetitionType.OPTIONAL + repeated = NRMD.FieldRepetitionType.REPEATED + projectionelements = NRMD.SchemaElement[ + nrroot(1), + nrelement("record"; repetition=required, children=Int32(2)), + nrelement("left"; repetition=required, children=Int32(1), + logical=nrlogical(:list)), + nrelement("list"; repetition=repeated, children=Int32(1)), + nrelement("element"; physical=NRMD.Type.INT32, repetition=required), + nrelement("right"; repetition=required, children=Int32(1), + logical=nrlogical(:list)), + nrelement("list"; repetition=repeated, children=Int32(1)), + nrelement("element"; physical=NRMD.Type.INT32, repetition=required), + ] + projectionplan = nrplan(projectionelements) + left = nrstream(projectionplan, 1, [0, 1, 0], [1, 1, 0], + Int32[1, 2]; rows=2) + right = nrstream(projectionplan, 2, [0, 0, 1], [1, 1, 1], + Int32[10, 20, 30]; rows=2) + projected = Parquet._assemblenested(projectionplan, [left, right], 2) + @test collect(projected[1]["record"]["left"]) == Int32[1, 2] + @test collect(projected[1]["record"]["right"]) == Int32[10] + @test isempty(projected[2]["record"]["left"]) + @test collect(projected[2]["record"]["right"]) == Int32[20, 30] + + nullelements = NRMD.SchemaElement[ + nrroot(1), + nrelement("record"; repetition=required, children=Int32(2)), + nrelement("day"; physical=NRMD.Type.INT32, repetition=optional, + logical=nrlogical(:date)), + nrelement("name"; physical=NRMD.Type.BYTE_ARRAY, repetition=optional, + logical=nrlogical(:string)), + ] + nullplan = nrplan(nullelements) + day = nrstream(nullplan, 1, [0], [0], Int32[]; rows=1) + name = nrstream(nullplan, 2, [0], [0], Vector{UInt8}[]; rows=1) + allnull = Parquet._assemblenested(nullplan, [day, name], 1) + @test allnull[1]["record"] isa Parquet.StructValue + @test allnull[1]["record"]["day"] === missing + @test allnull[1]["record"]["name"] === missing + @test eltype(allnull.children[1].children[1]) == Union{Missing,Dates.Date} + @test eltype(allnull.children[1].children[2]) == Union{Missing,String} +end + +@testset "nested reader metadata depth 128" begin + required = NRMD.FieldRepetitionType.REQUIRED + repeated = NRMD.FieldRepetitionType.REPEATED + elements = NRMD.SchemaElement[ + nrroot(1), + nrelement("outer"; repetition=required, children=Int32(1), + logical=nrlogical(:list)), + ] + for depth in 1:125 + push!(elements, nrelement("group_$depth"; repetition=repeated, + children=Int32(1), logical=nrlogical(:list))) + end + push!(elements, nrelement("element"; physical=NRMD.Type.INT32, + repetition=repeated)) + plan = nrplan(elements) + leaf = plan.leaves[1].source + @test length(elements) == 128 + @test length(leaf.path) == 127 + @test leaf.max_definition_level == Int16(126) + @test leaf.max_repetition_level == Int16(126) + stream = nrstream(plan, 1, [0], [leaf.max_definition_level], + Int32[1]; rows=1) + result = Parquet._assemblenested(plan, [stream], 1) + column = result.children[1] + typerepr = sprint(show, typeof(column)) + @test count("ListVector", typerepr) == 1 + @test ncodeunits(typerepr) < 16 * 1024 + value = result[1]["outer"] + for _ in 1:126 + value = only(value) + end + @test value == Int32(1) +end + +@testset "nested reader raw and logical byte ownership" begin + required = NRMD.FieldRepetitionType.REQUIRED + elements = NRMD.SchemaElement[ + nrroot(2), + nrelement("raw"; physical=NRMD.Type.BYTE_ARRAY, repetition=required), + nrelement("text"; physical=NRMD.Type.BYTE_ARRAY, repetition=required, + logical=nrlogical(:string)), + ] + plan = nrplan(elements) + rawbytes = UInt8[0x61] + textbytes = UInt8[0x62] + raw = nrstream(plan, 1, [0], [0], [rawbytes]; rows=1) + text = nrstream(plan, 2, [0], [0], [textbytes]; rows=1) + result = Parquet._assemblenested(plan, [raw, text], 1) + @test result[1]["raw"] === rawbytes + @test result[1]["text"] == "b" + rawbytes[1] = 0x63 + textbytes[1] = 0x64 + @test result[1]["raw"] == UInt8[0x63] + @test result[1]["text"] == "b" +end + +@testset "deep plans are rejected before recursive assembly" begin + # A plan deeper than the reader's recursion cap must fail with a clean + # LimitError instead of overflowing the stack. The guard is checked before + # any recursion, so this test never recurses to the cap depth itself. + maxdepth = Parquet._NESTED_READ_MAX_DEPTH + @test maxdepth == 1024 + required = NRMD.FieldRepetitionType.REQUIRED + limits = Parquet.Limits(max_metadata_depth=maxdepth + 8, + max_container_elements=maxdepth + 8) + function nrdeepchain(depth) + elements = NRMD.SchemaElement[nrroot(1)] + for _ in 1:(depth - 2) + push!(elements, nrelement("group"; repetition=required, + children=Int32(1))) + end + push!(elements, nrelement("leaf"; physical=NRMD.Type.INT32, + repetition=required)) + return nrplan(elements; limits=limits) + end + deepplan = nrdeepchain(maxdepth + 1) + @test deepplan.depth == maxdepth + 1 + deepstreams = Parquet.LeafStream[ + nrstream(deepplan, 1, [0], [0], Int32[7]; rows=1)] + deeperror = try + Parquet._assemblenested(deepplan, deepstreams, 1; limits=limits) + nothing + catch err + err + end + @test deeperror isa Parquet.LimitError + @test deeperror.resource == :nested_read_depth + + # A plan at the cap still assembles (moderate depth exercises the recursion). + okplan = nrdeepchain(64) + @test okplan.depth <= maxdepth + result = Parquet._assemblenested(okplan, + Parquet.LeafStream[nrstream(okplan, 1, [0], [0], Int32[7]; rows=1)], 1; + limits=limits) + @test length(result) == 1 +end diff --git a/test/nested_schema.jl b/test/nested_schema.jl new file mode 100644 index 0000000..a714ee4 --- /dev/null +++ b/test/nested_schema.jl @@ -0,0 +1,813 @@ +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end +if !isdefined(Parquet, :_NestedSchemaPlan) + Base.include(Parquet, joinpath(@__DIR__, "..", "src", "nested_schema.jl")) +end + +function nestedschemaelement(name; physical=nothing, repetition=nothing, + children=nothing, logical=nothing, converted=nothing) + return MD.SchemaElement(name=name, type_=physical, + repetition_type=repetition, num_children=children, + logicalType=logical, converted_type=converted) +end + +function nestedroot(children; logical=nothing, converted=nothing) + return nestedschemaelement("schema"; children=Int32(children), + logical=logical, converted=converted) +end + +function nestedlogical(kind::Symbol) + kind === :list && return MD.LogicalType(LIST=MD.ListType()) + kind === :map && return MD.LogicalType(MAP=MD.MapType()) + kind === :string && return MD.LogicalType(STRING=MD.StringType()) + kind === :variant && return MD.LogicalType( + VARIANT=MD.VariantType(specification_version=Int8(1))) + kind === :empty && return MD.LogicalType() + kind === :future && return MD.LogicalType( + unknown_fields=(TH.RawField(2555, TH.STRUCT, UInt8[0x00]),)) + throw(ArgumentError("unknown nested test logical type $kind")) +end + +function nestedcompile(elements; limits=Parquet.Limits()) + schema = Parquet.Schema(elements) + return Parquet._nestedplan(schema; limits=limits) +end + +@testset "nested canonical plans and thresholds" begin + required = MD.FieldRepetitionType.REQUIRED + optional = MD.FieldRepetitionType.OPTIONAL + repeated = MD.FieldRepetitionType.REPEATED + elements = MD.SchemaElement[ + nestedroot(2), + nestedschemaelement("names"; repetition=optional, children=Int32(1), + logical=nestedlogical(:list), converted=MD.ConvertedType.LIST), + nestedschemaelement("list"; repetition=repeated, children=Int32(1)), + nestedschemaelement("element"; physical=MD.Type.BYTE_ARRAY, + repetition=optional), + nestedschemaelement("record"; repetition=required, children=Int32(2)), + nestedschemaelement("id"; physical=MD.Type.INT64, repetition=required), + nestedschemaelement("score"; physical=MD.Type.DOUBLE, + repetition=optional), + ] + plan = nestedcompile(elements) + @test plan.source isa Parquet.Schema + @test plan.root.parent_definition == 0 + @test plan.root.present_definition == 0 + @test plan.root.leaf_range == Int32(1):Int32(3) + @test plan.plan_count == 6 + @test length(plan.leaves) == 3 + + list = plan.root.children[1] + @test list isa Parquet._NestedListPlan + @test list.annotation === :modern_list + @test list.rule == 0x06 + @test list.parent_definition == 0 + @test list.present_definition == 1 + @test list.entry_definition == 2 + @test list.repetition_level == 1 + @test list.leaf_range == Int32(1):Int32(1) + @test list.element isa Parquet._NestedLeafPlan + @test list.element.parent_definition == 2 + @test list.element.present_definition == 3 + + record = plan.root.children[2] + @test record isa Parquet._NestedStructPlan + @test record.parent_definition == 0 + @test record.present_definition == 0 + @test record.leaf_range == Int32(2):Int32(3) + @test record.children[1].parent_definition == 0 + @test record.children[1].present_definition == 0 + @test record.children[2].parent_definition == 0 + @test record.children[2].present_definition == 1 +end + +@testset "nested logical annotation precedence and placement" begin + required = MD.FieldRepetitionType.REQUIRED + repeated = MD.FieldRepetitionType.REPEATED + + modernlist = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("values"; repetition=required, children=Int32(1), + logical=nestedlogical(:list), converted=MD.ConvertedType.MAP), + nestedschemaelement("items"; physical=MD.Type.INT32, + repetition=repeated), + ]) + @test modernlist.root.children[1] isa Parquet._NestedListPlan + @test modernlist.root.children[1].annotation === :modern_list + + modernmap = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("values"; repetition=required, children=Int32(1), + logical=nestedlogical(:map), converted=MD.ConvertedType.LIST), + nestedschemaelement("pairs"; repetition=repeated, children=Int32(1)), + nestedschemaelement("first"; physical=MD.Type.INT64, + repetition=required), + ]) + @test modernmap.root.children[1] isa Parquet._NestedMapPlan + @test modernmap.root.children[1].annotation === :modern_map + + for modern in (:future, :empty, :variant) + ordinary = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("values"; repetition=required, + children=Int32(1), logical=nestedlogical(modern), + converted=MD.ConvertedType.LIST), + nestedschemaelement("items"; physical=MD.Type.INT32, + repetition=repeated), + ]) + @test ordinary.root.children[1] isa Parquet._NestedStructPlan + @test ordinary.root.children[1].children[1] isa Parquet._NestedListPlan + @test ordinary.root.children[1].children[1].annotation === + :unannotated_repeated + end + + unknownmap = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("values"; repetition=required, children=Int32(1), + logical=nestedlogical(:future), + converted=MD.ConvertedType.MAP_KEY_VALUE), + nestedschemaelement("items"; physical=MD.Type.INT32, + repetition=repeated), + ]) + @test unknownmap.root.children[1] isa Parquet._NestedStructPlan + + for converted in (MD.ConvertedType.LIST, MD.ConvertedType.MAP, + MD.ConvertedType.MAP_KEY_VALUE) + blocked = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("values"; repetition=required, + children=Int32(1), logical=nestedlogical(:future), + converted=converted), + nestedschemaelement("items"; physical=MD.Type.INT32, + repetition=repeated), + ]) + @test blocked.root.children[1] isa Parquet._NestedStructPlan + @test blocked.root.children[1].children[1] isa Parquet._NestedListPlan + end + + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("bad"; repetition=required, children=Int32(1), + logical=nestedlogical(:string), converted=MD.ConvertedType.LIST), + nestedschemaelement("item"; physical=MD.Type.INT32, + repetition=repeated), + ]) + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("bad"; physical=MD.Type.INT32, + repetition=required, logical=nestedlogical(:list)), + ]) + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("bad"; physical=MD.Type.INT32, + repetition=required, logical=nestedlogical(:future), + converted=MD.ConvertedType.MAP), + ]) + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(0; logical=nestedlogical(:list)), + ]) + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(0; logical=nestedlogical(:future), + converted=MD.ConvertedType.MAP_KEY_VALUE), + ]) +end + +@testset "all ordered LIST compatibility rules" begin + required = MD.FieldRepetitionType.REQUIRED + optional = MD.FieldRepetitionType.OPTIONAL + repeated = MD.FieldRepetitionType.REPEATED + list = nestedlogical(:list) + elements = MD.SchemaElement[ + nestedroot(6), + nestedschemaelement("rule1"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("scalar"; physical=MD.Type.INT32, + repetition=repeated), + nestedschemaelement("rule2"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("record"; repetition=repeated, children=Int32(2)), + nestedschemaelement("left"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("right"; physical=MD.Type.INT64, + repetition=required), + nestedschemaelement("rule3"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("nested"; repetition=repeated, children=Int32(1), + logical=list), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=repeated), + nestedschemaelement("rule4"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("array"; repetition=repeated, children=Int32(1)), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("rule5"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("rule5_tuple"; repetition=repeated, + children=Int32(1)), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("rule6"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("items"; repetition=repeated, children=Int32(1)), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=optional), + ] + plan = nestedcompile(elements) + lists = plan.root.children + @test [item.rule for item in lists] == UInt8[1, 2, 3, 4, 5, 6] + @test [item.leaf_range for item in lists] == UnitRange{Int32}[ + Int32(1):Int32(1), Int32(2):Int32(3), Int32(4):Int32(4), + Int32(5):Int32(5), Int32(6):Int32(6), Int32(7):Int32(7)] + @test lists[1].element isa Parquet._NestedLeafPlan + @test lists[2].element isa Parquet._NestedStructPlan + @test lists[3].element isa Parquet._NestedListPlan + @test lists[3].element.annotation === :modern_list + @test lists[3].element.parent_definition == lists[3].entry_definition + @test lists[4].element isa Parquet._NestedStructPlan + @test lists[5].element isa Parquet._NestedStructPlan + @test lists[6].element isa Parquet._NestedLeafPlan + @test lists[6].element.parent_definition == lists[6].entry_definition + @test lists[6].element.present_definition == + lists[6].entry_definition + Int16(1) + + ordinaryrule3 = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("outer"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("wrapper"; repetition=repeated, children=Int32(1)), + nestedschemaelement("inner"; physical=MD.Type.INT32, + repetition=repeated), + ]).root.children[1] + @test ordinaryrule3.rule == 0x03 + @test ordinaryrule3.element isa Parquet._NestedStructPlan + @test ordinaryrule3.element.children[1] isa Parquet._NestedListPlan + + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("empty"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("array"; repetition=repeated, children=Int32(0)), + ]) +end + +@testset "malformed LIST schemas" begin + required = MD.FieldRepetitionType.REQUIRED + optional = MD.FieldRepetitionType.OPTIONAL + repeated = MD.FieldRepetitionType.REPEATED + list = nestedlogical(:list) + malformed = Vector{MD.SchemaElement}[] + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("list"; repetition=required, children=Int32(0), + logical=list), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("list"; repetition=required, children=Int32(2), + logical=list), + nestedschemaelement("one"; physical=MD.Type.INT32, + repetition=repeated), + nestedschemaelement("two"; physical=MD.Type.INT32, + repetition=repeated), + ]) + for entryrepetition in (required, optional) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("list"; repetition=required, + children=Int32(1), logical=list), + nestedschemaelement("entry"; physical=MD.Type.INT32, + repetition=entryrepetition), + ]) + end + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("list"; repetition=required, children=Int32(1), + logical=list), + nestedschemaelement("wrapper"; repetition=repeated, + children=Int32(1), converted=MD.ConvertedType.MAP), + nestedschemaelement("only"; physical=MD.Type.INT32, + repetition=required), + ]) + for elements in malformed + @test_throws Parquet.FormatError nestedcompile(elements) + end +end + +@testset "repeated annotated collection exceptions" begin + required = MD.FieldRepetitionType.REQUIRED + repeated = MD.FieldRepetitionType.REPEATED + for annotation in (:list, :map) + elements = if annotation === :list + MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("bad"; repetition=repeated, + children=Int32(1), logical=nestedlogical(:list)), + nestedschemaelement("items"; physical=MD.Type.INT32, + repetition=repeated), + ] + else + MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("bad"; repetition=repeated, + children=Int32(1), logical=nestedlogical(:map)), + nestedschemaelement("pairs"; repetition=repeated, + children=Int32(1)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + ] + end + @test_throws Parquet.FormatError nestedcompile(elements) + end + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("bad"; repetition=repeated, children=Int32(1), + converted=MD.ConvertedType.MAP_KEY_VALUE), + nestedschemaelement("pairs"; repetition=repeated, children=Int32(1)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + ]) + + listmap = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("maps"; repetition=required, children=Int32(1), + converted=MD.ConvertedType.LIST), + nestedschemaelement("map"; repetition=repeated, children=Int32(1), + converted=MD.ConvertedType.MAP), + nestedschemaelement("entries"; repetition=repeated, children=Int32(2), + converted=MD.ConvertedType.MAP_KEY_VALUE), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("value"; physical=MD.Type.INT64, + repetition=required), + ]) + outer = listmap.root.children[1] + @test outer isa Parquet._NestedListPlan + @test outer.rule == 0x03 + @test outer.annotation === :legacy_list + @test outer.element isa Parquet._NestedMapPlan + @test outer.element.annotation === :legacy_map + @test outer.element.parent_definition == outer.entry_definition + @test outer.element.entry_has_map_key_value + + listalias = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("maps"; repetition=required, children=Int32(1), + logical=nestedlogical(:list)), + nestedschemaelement("map"; repetition=repeated, children=Int32(1), + converted=MD.ConvertedType.MAP_KEY_VALUE), + nestedschemaelement("entries"; repetition=repeated, + children=Int32(1)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + ]) + @test listalias.root.children[1].element isa Parquet._NestedMapPlan + @test listalias.root.children[1].element.annotation === + :legacy_map_key_value +end + +@testset "MAP compatibility forms" begin + required = MD.FieldRepetitionType.REQUIRED + optional = MD.FieldRepetitionType.OPTIONAL + repeated = MD.FieldRepetitionType.REPEATED + elements = MD.SchemaElement[ + nestedroot(4), + nestedschemaelement("canonical"; repetition=optional, children=Int32(1), + logical=nestedlogical(:map), converted=MD.ConvertedType.MAP), + nestedschemaelement("anything"; repetition=repeated, children=Int32(2)), + nestedschemaelement("not_key"; physical=MD.Type.BYTE_ARRAY, + repetition=required), + nestedschemaelement("not_value"; physical=MD.Type.INT32, + repetition=optional), + nestedschemaelement("legacy"; repetition=required, children=Int32(1), + converted=MD.ConvertedType.MAP), + nestedschemaelement("pairs"; repetition=repeated, children=Int32(2), + converted=MD.ConvertedType.MAP_KEY_VALUE), + nestedschemaelement("left"; physical=MD.Type.INT64, + repetition=optional), + nestedschemaelement("right"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("alias"; repetition=optional, children=Int32(1), + converted=MD.ConvertedType.MAP_KEY_VALUE), + nestedschemaelement("pairs"; repetition=repeated, children=Int32(1)), + nestedschemaelement("only"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("future_entry"; repetition=required, + children=Int32(1), logical=nestedlogical(:map)), + nestedschemaelement("pairs"; repetition=repeated, children=Int32(1), + logical=nestedlogical(:future), + converted=MD.ConvertedType.MAP_KEY_VALUE), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + ] + plan = nestedcompile(elements) + canonical, legacy, alias, futureentry = plan.root.children + + @test canonical isa Parquet._NestedMapPlan + @test canonical.annotation === :modern_map + @test canonical.parent_definition == 0 + @test canonical.present_definition == 1 + @test canonical.entry_definition == 2 + @test canonical.repetition_level == 1 + @test !canonical.optional_key + @test !canonical.entry_has_map_key_value + @test canonical.key.source.element.name == "not_key" + @test canonical.value.source.element.name == "not_value" + @test canonical.key.parent_definition == 2 + @test canonical.key.present_definition == 2 + @test canonical.value.parent_definition == 2 + @test canonical.value.present_definition == 3 + + @test legacy.annotation === :legacy_map + @test legacy.optional_key + @test legacy.entry_has_map_key_value + @test legacy.key.present_definition == 2 + @test legacy.value.present_definition == 1 + @test alias.annotation === :legacy_map_key_value + @test alias.value === nothing + @test !alias.optional_key + @test futureentry.annotation === :modern_map + @test !futureentry.entry_has_map_key_value + @test plan.root.leaf_range == Int32(1):Int32(6) +end + +@testset "MAP malformed schemas" begin + required = MD.FieldRepetitionType.REQUIRED + optional = MD.FieldRepetitionType.OPTIONAL + repeated = MD.FieldRepetitionType.REPEATED + map = nestedlogical(:map) + + malformed = Vector{MD.SchemaElement}[] + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(0), + logical=map), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(2), + logical=map), + nestedschemaelement("one"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("two"; physical=MD.Type.INT32, + repetition=required), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; physical=MD.Type.INT32, + repetition=repeated), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; repetition=required, children=Int32(1)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; repetition=repeated, children=Int32(0)), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; repetition=repeated, children=Int32(3)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=optional), + nestedschemaelement("extra"; physical=MD.Type.INT32, + repetition=optional), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; repetition=repeated, children=Int32(1)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=repeated), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; repetition=repeated, children=Int32(2)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=repeated), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; repetition=repeated, children=Int32(1), + converted=MD.ConvertedType.LIST), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + ]) + push!(malformed, MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("map"; repetition=required, children=Int32(1), + logical=map), + nestedschemaelement("entry"; repetition=repeated, children=Int32(1), + logical=nestedlogical(:map)), + nestedschemaelement("key"; physical=MD.Type.INT32, + repetition=required), + ]) + for elements in malformed + @test_throws Parquet.FormatError nestedcompile(elements) + end +end + +@testset "unannotated repeated fields and duplicate names" begin + required = MD.FieldRepetitionType.REQUIRED + optional = MD.FieldRepetitionType.OPTIONAL + repeated = MD.FieldRepetitionType.REPEATED + plan = nestedcompile(MD.SchemaElement[ + nestedroot(3), + nestedschemaelement("numbers"; physical=MD.Type.INT32, + repetition=repeated), + nestedschemaelement("records"; repetition=repeated, children=Int32(2)), + nestedschemaelement("same"; physical=MD.Type.INT32, + repetition=required), + nestedschemaelement("same"; physical=MD.Type.INT64, + repetition=optional), + nestedschemaelement("holder"; repetition=required, children=Int32(2)), + nestedschemaelement("same"; physical=MD.Type.FLOAT, + repetition=required), + nestedschemaelement("same"; physical=MD.Type.DOUBLE, + repetition=required), + ]) + numbers, records, holder = plan.root.children + @test numbers isa Parquet._NestedListPlan + @test numbers.annotation === :unannotated_repeated + @test numbers.rule == 0x00 + @test numbers.parent_definition == 0 + @test numbers.present_definition == 0 + @test numbers.entry_definition == 1 + @test numbers.element.parent_definition == 1 + @test numbers.element.present_definition == 1 + + @test records isa Parquet._NestedListPlan + @test records.element isa Parquet._NestedStructPlan + @test [child.source.element.name for child in records.element.children] == + ["same", "same"] + @test records.leaf_range == Int32(2):Int32(3) + @test holder isa Parquet._NestedStructPlan + @test [child.source.element.name for child in holder.children] == + ["same", "same"] + @test holder.leaf_range == Int32(4):Int32(5) + @test [leaf.source.column_index for leaf in plan.leaves] == + Int32[1, 2, 3, 4, 5] +end + +@testset "leafless groups and plan limits" begin + required = MD.FieldRepetitionType.REQUIRED + optional = MD.FieldRepetitionType.OPTIONAL + repeated = MD.FieldRepetitionType.REPEATED + + emptyroot = nestedcompile(MD.SchemaElement[nestedroot(0)]) + @test isempty(emptyroot.root.children) + @test isempty(emptyroot.root.leaf_range) + @test isempty(emptyroot.leaves) + + onlyrequired = nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("empty"; repetition=required, children=Int32(0)), + ]) + @test onlyrequired.root.children[1] isa Parquet._NestedStructPlan + @test isempty(onlyrequired.root.children[1].leaf_range) + @test isempty(onlyrequired.leaves) + + requiredempty = nestedcompile(MD.SchemaElement[ + nestedroot(2), + nestedschemaelement("empty"; repetition=required, children=Int32(0)), + nestedschemaelement("anchor"; physical=MD.Type.INT32, + repetition=required), + ]) + @test requiredempty.root.children[1] isa Parquet._NestedStructPlan + @test isempty(requiredempty.root.children[1].leaf_range) + @test requiredempty.root.leaf_range == Int32(1):Int32(1) + + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("empty"; repetition=optional, children=Int32(0)), + ]) + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("empty"; repetition=repeated, children=Int32(0)), + ]) + @test_throws Parquet.FormatError nestedcompile(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("outer"; repetition=optional, children=Int32(1)), + nestedschemaelement("empty"; repetition=required, children=Int32(0)), + ]) + + schema = Parquet.Schema(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=required), + ]) + error = try + Parquet._nestedplan(schema; + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :container_elements + @test error.requested == 2 + @test error.maximum == 1 +end + +function nestedmanualchain(depth::Int) + depth >= 2 || throw(ArgumentError("manual schema depth must be at least two")) + required = MD.FieldRepetitionType.REQUIRED + path = String[] + leaf = Parquet.SchemaNode(nestedschemaelement("leaf"; + physical=MD.Type.INT32, repetition=required), path, Int16(0), + Int16(0), Int32(1), Parquet.SchemaNode[]) + node = leaf + for _ in 1:(depth - 2) + node = Parquet.SchemaNode(nestedschemaelement("group"; + repetition=required, children=Int32(1)), path, Int16(0), + Int16(0), Int32(0), Parquet.SchemaNode[node]) + end + root = Parquet.SchemaNode(nestedroot(1), path, Int16(0), Int16(0), + Int32(0), Parquet.SchemaNode[node]) + return Parquet.Schema(root, Parquet.SchemaNode[leaf]) +end + +@testset "iterative nested limits and rollback" begin + required = MD.FieldRepetitionType.REQUIRED + schema = Parquet.Schema(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("value"; physical=MD.Type.INT32, + repetition=required), + ]) + exactdepth = Parquet._nestedplan(schema; + limits=Parquet.Limits(max_metadata_depth=2, + max_container_elements=2)) + @test exactdepth.plan_count == 2 + retainedlimits = Parquet.Limits() + retainedbudget = Parquet._LiveByteBudget(retainedlimits) + Parquet._reserve!(retainedbudget, 64) + retained = Parquet._nestedplan(schema; limits=retainedlimits, + budget=retainedbudget) + expectedretained = + Parquet._materializedarraybytes(Parquet._NestedLeafPlan, 1) + + Parquet._materializedarraybytes(Parquet._NestedPlan, 1) + + 3 * Parquet._MATERIALIZED_OBJECT_BYTES + @test retained.plan_count == 2 + @test Parquet._budgetused(retainedbudget) == 64 + expectedretained + deptherror = try + Parquet._nestedplan(schema; + limits=Parquet.Limits(max_metadata_depth=1)) + nothing + catch err + err + end + @test deptherror isa Parquet.LimitError + @test deptherror.resource == :metadata_depth + @test deptherror.requested == 2 + @test deptherror.maximum == 1 + + counterror = try + Parquet._nestedplan(schema; + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test counterror isa Parquet.LimitError + @test counterror.resource == :container_elements + @test counterror.requested == 2 + @test counterror.maximum == 1 + + path = String[] + malformedleaf = Parquet.SchemaNode(nestedschemaelement("bad"; + physical=MD.Type.INT32), path, Int16(0), Int16(0), Int32(1), + Parquet.SchemaNode[]) + malformedroot = Parquet.SchemaNode(nestedroot(1), path, Int16(0), + Int16(0), Int32(0), Parquet.SchemaNode[malformedleaf]) + malformed = Parquet.Schema(malformedroot, + Parquet.SchemaNode[malformedleaf]) + limits = Parquet.Limits(max_metadata_depth=1, + max_container_elements=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + malformederror = try + Parquet._nestedplan(malformed; limits=limits, budget=budget) + nothing + catch err + err + end + @test malformederror isa Parquet.FormatError + @test occursin("no valid repetition type", malformederror.message) + @test Parquet._budgetused(budget) == 64 + + listschema = Parquet.Schema(MD.SchemaElement[ + nestedroot(1), + nestedschemaelement("values"; repetition=required, + children=Int32(1), logical=nestedlogical(:list)), + nestedschemaelement("array"; repetition=MD.FieldRepetitionType.REPEATED, + children=Int32(0)), + ]) + precedencebudget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(precedencebudget, 64) + precedenceerror = try + Parquet._nestedplan(listschema; limits=limits, + budget=precedencebudget) + nothing + catch err + err + end + @test precedenceerror isa Parquet.FormatError + @test occursin("zero-field repeated wrapper", precedenceerror.message) + @test Parquet._budgetused(precedencebudget) == 64 + + firstleaf = Parquet.SchemaNode(nestedschemaelement("first"; + physical=MD.Type.INT32, repetition=required), path, Int16(0), + Int16(0), Int32(1), Parquet.SchemaNode[]) + secondleaf = Parquet.SchemaNode(nestedschemaelement("second"; + physical=MD.Type.INT32, repetition=required), path, Int16(0), + Int16(0), Int32(0), Parquet.SchemaNode[]) + partialroot = Parquet.SchemaNode(nestedroot(2), path, Int16(0), Int16(0), + Int32(0), Parquet.SchemaNode[firstleaf, secondleaf]) + partial = Parquet.Schema(partialroot, + Parquet.SchemaNode[firstleaf, secondleaf]) + partiallimits = Parquet.Limits() + partialbudget = Parquet._LiveByteBudget(partiallimits) + Parquet._reserve!(partialbudget, 64) + @test_throws Parquet.FormatError Parquet._nestedplan(partial; + limits=partiallimits, budget=partialbudget) + @test Parquet._budgetused(partialbudget) == 64 + + aliasedroot = Parquet.SchemaNode(nestedroot(2), path, Int16(0), Int16(0), + Int32(0), Parquet.SchemaNode[firstleaf, firstleaf]) + aliased = Parquet.Schema(aliasedroot, Parquet.SchemaNode[firstleaf]) + aliasbudget = Parquet._LiveByteBudget(partiallimits) + Parquet._reserve!(aliasbudget, 64) + aliaserror = try + Parquet._nestedplan(aliased; limits=partiallimits, + budget=aliasbudget) + nothing + catch err + err + end + @test aliaserror isa Parquet.FormatError + @test occursin("more physical leaf occurrences", aliaserror.message) + @test Parquet._budgetused(aliasbudget) == 64 +end + +@testset "50,000-node iterative nested plan" begin + depth = 50_000 + schema = nestedmanualchain(depth) + limits = Parquet.Limits(max_metadata_depth=depth, + max_container_elements=depth, max_materialized_bytes=128 * 1024 * 1024) + plan = Parquet._nestedplan(schema; limits=limits) + @test plan.plan_count == depth + @test plan.root.leaf_range == Int32(1):Int32(1) + current::Parquet._NestedPlan = plan.root + visited = 1 + singlechild = true + while current isa Parquet._NestedStructPlan + if length(current.children) != 1 + singlechild = false + break + end + current = current.children[1] + visited += 1 + end + @test singlechild + @test current isa Parquet._NestedLeafPlan + @test visited == depth + + faillimits = Parquet.Limits(max_metadata_depth=depth - 1, + max_container_elements=depth, max_materialized_bytes=128 * 1024 * 1024) + failbudget = Parquet._LiveByteBudget(faillimits) + Parquet._reserve!(failbudget, 64) + deptherror = try + Parquet._nestedplan(schema; limits=faillimits, budget=failbudget) + nothing + catch err + err + end + @test deptherror isa Parquet.LimitError + @test deptherror.resource == :metadata_depth + @test deptherror.requested == depth + @test deptherror.maximum == depth - 1 + @test Parquet._budgetused(failbudget) == 64 +end diff --git a/test/nested_table.jl b/test/nested_table.jl new file mode 100644 index 0000000..c553d17 --- /dev/null +++ b/test/nested_table.jl @@ -0,0 +1,370 @@ +using Test + +const NTMD = Parquet.Metadata +const NESTED_TABLE_CORPUS = get(ENV, "PARQUET_TESTING_DIR", + joinpath(@__DIR__, "parquet-testing")) +const NESTED_TABLE_FIXTURES = ( + "list_columns.parquet", + "null_list.parquet", + "datapage_v2.snappy.parquet", + "old_list_structure.parquet", + "nested_lists.snappy.parquet", + "nested_maps.snappy.parquet", + "repeated_primitive_no_list.parquet", + "repeated_no_annotation.parquet", + "nullable.impala.parquet", + "nonnullable.impala.parquet", + "map_no_value.parquet", + "incorrect_map_schema.parquet", + "nested_structs.rust.parquet", +) + +function ntfixture(name::AbstractString) + return joinpath(NESTED_TABLE_CORPUS, "data", name) +end + +function ntreadroot(input; limits::Parquet.Limits=Parquet.Limits()) + budget = Parquet._LiveByteBudget(limits) + file = Parquet.File(input; limits=limits, budget=budget) + try + metadata = Parquet._readfilemetadata(file, limits, budget) + schema = Parquet.Schema(metadata; limits=limits, budget=budget) + plan = Parquet._nestedplan(schema; limits=limits, budget=budget) + root = Parquet._readnestedroot(file, metadata, schema, plan, limits, + budget) + return (; root, metadata, schema, plan, budget) + finally + close(file) + end +end + +function ntsemantic(value) + ismissing(value) && return missing + if value isa Parquet.StructValue + output = Pair{String,Any}[] + sizehint!(output, length(value)) + for index in 1:length(value) + push!(output, value.names[index] => ntsemantic(value[index])) + end + return output + end + if value isa Parquet.MapValue + output = Pair{Any,Any}[] + sizehint!(output, length(value)) + for pair in value + push!(output, ntsemantic(pair.first) => ntsemantic(pair.second)) + end + return output + end + if value isa Parquet.ListValue + output = [] + sizehint!(output, length(value)) + for item in value + push!(output, ntsemantic(item)) + end + return output + end + return value +end + +function ntreplace(value; replacements...) + names = fieldnames(typeof(value)) + values = map(names) do name + return haskey(replacements, name) ? replacements[name] : + getfield(value, name) + end + return typeof(value)(values...) +end + +function ntshift(offset::Nothing, delta::Int64) + return nothing +end + +function ntshift(offset::Int64, delta::Int64) + return Base.checked_add(offset, delta) +end + +function ntshiftcolumn(chunk::NTMD.ColumnChunk, delta::Int64) + md = something(chunk.meta_data) + shiftedmd = ntreplace(md; + data_page_offset=ntshift(md.data_page_offset, delta), + index_page_offset=ntshift(md.index_page_offset, delta), + dictionary_page_offset=ntshift(md.dictionary_page_offset, delta), + bloom_filter_offset=ntshift(md.bloom_filter_offset, delta), + ) + return ntreplace(chunk; + file_offset=ntshift(chunk.file_offset, delta), + meta_data=shiftedmd, + offset_index_offset=ntshift(chunk.offset_index_offset, delta), + column_index_offset=ntshift(chunk.column_index_offset, delta), + ) +end + +function ntshiftrowgroup(group::NTMD.RowGroup, delta::Int64) + columns = NTMD.ColumnChunk[ + ntshiftcolumn(column, delta) for column in group.columns] + ordinal = group.ordinal === nothing ? nothing : + Base.checked_add(group.ordinal, Int16(1)) + return ntreplace(group; columns=columns, + file_offset=ntshift(group.file_offset, delta), ordinal=ordinal) +end + +function ntduplicaterowgroup(path::AbstractString) + source = read(path) + file = Parquet.File(source) + metadata = try + Parquet.Thrift.decode(file.footer.bytes, NTMD.FileMetaData) + finally + close(file) + end + length(metadata.row_groups) == 1 || throw(ArgumentError( + "nested row-group fixture must have one row group")) + footeroffset = Int64(length(source) - 8 - + Int(Parquet._readu32le(@view source[(end - 7):(end - 4)]))) + footeroffset >= 4 || throw(ArgumentError( + "nested row-group fixture has no data region")) + body = @view source[5:Int(footeroffset)] + delta = Int64(length(body)) + firstgroup = only(metadata.row_groups) + secondgroup = ntshiftrowgroup(firstgroup, delta) + duplicated = ntreplace(metadata; + num_rows=Base.checked_mul(metadata.num_rows, Int64(2)), + row_groups=NTMD.RowGroup[firstgroup, secondgroup], + ) + output = copy(source[1:Int(footeroffset)]) + append!(output, body) + footer = Parquet.Thrift.encode(duplicated) + append!(output, footer) + Parquet._writelittle!(output, UInt32(length(footer))) + append!(output, Parquet.PARQUET_MAGIC) + return output, duplicated, delta +end + +@testset "whole-table nested Apache corpus" begin + datadir = joinpath(NESTED_TABLE_CORPUS, "data") + if isdir(datadir) + @test all(name -> isfile(ntfixture(name)), NESTED_TABLE_FIXTURES) + + lists = ntreadroot(ntfixture("list_columns.parquet")) + @test lists.root.names == ["int64_list", "utf8_list"] + @test length(lists.root) == 3 + @test isequal(ntsemantic(lists.root[1]["int64_list"]), [1, 2, 3]) + @test isequal(ntsemantic(lists.root[2]["int64_list"]), [missing, 1]) + @test lists.root[2]["utf8_list"] === missing + @test isequal(ntsemantic(lists.root[3]["utf8_list"]), + ["efg", missing, "hij", "xyz"]) + + null_list = ntreadroot(ntfixture("null_list.parquet")).root + @test null_list.names == ["emptylist"] + @test length(null_list) == 1 + @test null_list[1]["emptylist"] !== missing + @test isempty(null_list[1]["emptylist"]) + + v2 = ntreadroot(ntfixture("datapage_v2.snappy.parquet")).root + @test v2.names == ["a", "b", "c", "d", "e"] + @test length(v2) == 5 + @test [v2[row]["b"] for row in 1:5] == Int64[1, 2, 3, 4, 5] + @test isequal([v2[row]["a"] for row in 1:5], + ["abc", "abc", "abc", missing, "abc"]) + @test isequal([ntsemantic(v2[row]["e"]) for row in 1:5], + Any[[1, 2, 3], missing, missing, [1, 2, 3], [1, 2]]) + + old = ntreadroot(ntfixture("old_list_structure.parquet")).root + @test old.names == ["a"] + @test length(old) == 1 + @test isequal(ntsemantic(old[1]["a"]), [[1, 2], [3, 4]]) + + nestedlists = ntreadroot( + ntfixture("nested_lists.snappy.parquet")).root + @test nestedlists.names == ["a", "b"] + @test length(nestedlists) == 3 + @test [nestedlists[row]["b"] for row in 1:3] == Int32[1, 1, 1] + @test isequal(ntsemantic(nestedlists[1]["a"]), + [[Any["a", "b"], Any["c"]], [missing, Any["d"]]]) + @test isequal(ntsemantic(nestedlists[3]["a"]), + [[Any["a", "b"], Any["c", "d"], Any["e"]], + [missing, Any["f"]]]) + + nestedmaps = ntreadroot( + ntfixture("nested_maps.snappy.parquet")).root + @test nestedmaps.names == ["a", "b", "c"] + @test length(nestedmaps) == 6 + @test [nestedmaps[row]["b"] for row in 1:6] == fill(Int32(1), 6) + @test [nestedmaps[row]["c"] for row in 1:6] == fill(1.0, 6) + @test isequal(ntsemantic(nestedmaps[1]["a"]), + ["a" => Any[1 => true, 2 => false]]) + @test isequal(ntsemantic(nestedmaps[3]["a"]), ["c" => missing]) + @test isequal(ntsemantic(nestedmaps[4]["a"]), ["d" => Any[]]) + @test isequal(ntsemantic(nestedmaps[6]["a"]), + ["f" => Any[3 => true, 4 => false, 5 => true]]) + + repeated = ntreadroot( + ntfixture("repeated_primitive_no_list.parquet")).root + @test repeated.names == + ["Int32_list", "String_list", "group_of_lists"] + @test length(repeated) == 4 + expectedints = Any[[0, 1, 2, 3], [], [4], [5, 6, 7, 8]] + expectedstrings = Any[["foo", "zero", "one", "two"], ["three"], + ["four"], ["five", "six", "seven", "eight"]] + @test isequal([ntsemantic(repeated[row]["Int32_list"]) + for row in 1:4], expectedints) + @test isequal([ntsemantic(repeated[row]["String_list"]) + for row in 1:4], expectedstrings) + @test all(row -> isequal( + ntsemantic(repeated[row]["Int32_list"]), + ntsemantic(repeated[row]["group_of_lists"]["Int32_list_in_group"])), + 1:4) + @test all(row -> isequal( + ntsemantic(repeated[row]["String_list"]), + ntsemantic(repeated[row]["group_of_lists"]["String_list_in_group"])), + 1:4) + + nullable = ntreadroot(ntfixture("nullable.impala.parquet")).root + @test nullable.names == ["id", "int_array", "int_array_Array", + "int_map", "int_Map_Array", "nested_struct"] + @test length(nullable) == 7 + @test isequal(ntsemantic(nullable[1]["int_array"]), [1, 2, 3]) + @test isempty(nullable[3]["int_array"]) + @test nullable[4]["int_array"] === missing + @test isempty(nullable[4]["int_array_Array"]) + @test nullable[5]["int_array_Array"] === missing + @test isempty(nullable[3]["int_map"]) + @test nullable[6]["int_map"] === missing + @test nullable[6]["nested_struct"] === missing + @test nullable[7]["nested_struct"]["A"] == 7 + @test isequal(ntsemantic(nullable[7]["nested_struct"]["b"]), + [2, 3, missing]) + + nonnullable = ntreadroot( + ntfixture("nonnullable.impala.parquet")).root + @test nonnullable.names == ["ID", "Int_Array", "int_array_array", + "Int_Map", "int_map_array", "nested_Struct"] + @test length(nonnullable) == 1 + @test nonnullable[1]["ID"] == 8 + @test isequal(ntsemantic(nonnullable[1]["int_array_array"]), + [[-1, -2], Any[]]) + @test isequal(ntsemantic(nonnullable[1]["Int_Map"]), ["k1" => -1]) + @test isempty(nonnullable[1]["nested_Struct"]["G"]) + + novalue = ntreadroot(ntfixture("map_no_value.parquet")).root + @test novalue.names == ["my_map", "my_map_no_v", "my_list"] + @test length(novalue) == 3 + for row in 1:3 + firstkey = 3 * row - 2 + expectedkeys = collect(Int32(firstkey):Int32(firstkey + 2)) + @test [pair.first for pair in novalue[row]["my_map"]] == + expectedkeys + @test all(pair -> ismissing(pair.second), + novalue[row]["my_map"]) + @test isequal(ntsemantic(novalue[row]["my_map"]), + ntsemantic(novalue[row]["my_map_no_v"])) + @test collect(novalue[row]["my_list"]) == expectedkeys + end + + incorrect = ntreadroot( + ntfixture("incorrect_map_schema.parquet")).root + @test incorrect.names == ["my_map"] + @test length(incorrect) == 1 + @test length(incorrect[1]["my_map"]) == 2 + @test Parquet.maplookup(incorrect[1]["my_map"], "name") == "report" + @test Parquet.maplookup(incorrect[1]["my_map"], "parent") == + "another" + + structs = ntreadroot(ntfixture("nested_structs.rust.parquet")).root + @test length(structs) == 1 + @test length(structs.names) == 36 + @test structs.names[1:4] == ["roll_num", "PC_CUR", "CVA_2012", + "CVA_2016"] + @test structs[1]["roll_num"]["min"] == 190406409000602 + @test structs[1]["PC_CUR"]["max"] == 742 + @test structs[1]["CVA_2012"]["max"] == 32150509 + @test structs[1]["CVA_2016"]["max"] == 35195000 + @test structs[1]["BIA_3"]["count"] == 0 + @test structs[1]["count"]["sum"] == 495 + else + @info "parquet-testing corpus not found; skipping whole-table nested fixtures" corpus=NESTED_TABLE_CORPUS + end +end + +@testset "whole-table nested row-group concatenation" begin + fixture = ntfixture("list_columns.parquet") + if isfile(fixture) + bytes, metadata, delta = ntduplicaterowgroup(fixture) + @test metadata.num_rows == 6 + @test length(metadata.row_groups) == 2 + firstgroup, secondgroup = metadata.row_groups + @test secondgroup.num_rows == firstgroup.num_rows == 3 + for (first, second) in zip(firstgroup.columns, secondgroup.columns) + @test second.file_offset == first.file_offset + delta + @test second.meta_data.data_page_offset == + first.meta_data.data_page_offset + delta + @test second.meta_data.dictionary_page_offset == + first.meta_data.dictionary_page_offset + delta + @test second.offset_index_offset === nothing + @test second.column_index_offset === nothing + @test second.meta_data.bloom_filter_offset === nothing + end + result = ntreadroot(bytes) + @test length(result.root) == 6 + @test [group.num_rows for group in result.metadata.row_groups] == [3, 3] + @test all(row -> isequal(ntsemantic(result.root[row]), + ntsemantic(result.root[row + 3])), 1:3) + @test isequal([ntsemantic(result.root[row]["int64_list"]) + for row in 1:6], + Any[[1, 2, 3], [missing, 1], [4], [1, 2, 3], [missing, 1], [4]]) + else + @info "list_columns fixture not found; skipping nested row-group concatenation" corpus=NESTED_TABLE_CORPUS + end +end + +@testset "legacy zero footer row-count sentinel" begin + fixture = ntfixture("repeated_no_annotation.parquet") + if isfile(fixture) + result = ntreadroot(fixture) + @test result.metadata.num_rows == 0 + @test [group.num_rows for group in result.metadata.row_groups] == [6] + @test length(result.root) == 6 + @test [result.root[row]["id"] for row in 1:6] == Int64[1, 2, 3, 4, 5, 6] + @test result.root[1]["phoneNumbers"] === missing + @test isempty(result.root[3]["phoneNumbers"]["phone"]) + @test result.root[4]["phoneNumbers"]["phone"][1]["number"] == + 5555555555 + @test [phone["number"] for phone in + result.root[6]["phoneNumbers"]["phone"]] == + Int64[1111111111, 2222222222, 3333333333] + @test result.root[6]["phoneNumbers"]["phone"][2]["kind"] === missing + else + @info "legacy sentinel fixture not found; skipping nested row-count test" corpus=NESTED_TABLE_CORPUS + end +end + +@testset "whole-table nested materialization cleanup" begin + fixture = ntfixture("list_columns.parquet") + if isfile(fixture) + setup = ntreadroot(fixture) + count = length(setup.plan.leaves) + initial = Parquet._materializedarraybytes(Parquet.LeafStream, count) + + Parquet._materializedarraybytes(Int, count) + + Parquet._materializedarraybytes(Int64, count) + limits = Parquet.Limits(max_materialized_bytes=initial) + budget = Parquet._LiveByteBudget(limits) + file = Parquet.File(fixture) + try + error = try + Parquet._readnestedroot(file, setup.metadata, setup.schema, + setup.plan, limits, budget) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :materialized_bytes + @test error.requested > error.maximum == initial + @test Parquet._budgetused(budget) == 0 + finally + close(file) + end + else + @info "list_columns fixture not found; skipping nested cleanup test" corpus=NESTED_TABLE_CORPUS + end +end diff --git a/test/nested_vectors.jl b/test/nested_vectors.jl new file mode 100644 index 0000000..f724c53 --- /dev/null +++ b/test/nested_vectors.jl @@ -0,0 +1,399 @@ +using Test + +struct VirtualNestedVector{T} <: AbstractVector{T} + length::Int +end + +struct UnsupportedNestedKey + values::Vector{Int} +end + +mutable struct MutableNestedKey + value::Int +end + +struct ThrowNestedMetric{E} <: AbstractVector{Int32} + error::E +end + +struct MutatingNestedVector{T,F} <: AbstractVector{T} + values::Vector{T} + action::F +end + +struct NonIntNestedLength <: AbstractVector{Int32} end + +function Base.IndexStyle(::Type{<:VirtualNestedVector}) + return IndexLinear() +end + +function Base.size(values::VirtualNestedVector) + return (values.length,) +end + +function Base.getindex(values::VirtualNestedVector{T}, index::Int) where {T} + @boundscheck checkbounds(values, index) + return zero(T) +end + +function Base.size(values::ThrowNestedMetric) + throw(values.error) +end + +function Base.getindex(::ThrowNestedMetric, ::Int) + return Int32(0) +end + +function Base.IndexStyle(::Type{<:MutatingNestedVector}) + return IndexLinear() +end + +function Base.size(values::MutatingNestedVector) + return size(values.values) +end + +function Base.getindex(values::MutatingNestedVector, index::Int) + item = values.values[index] + values.action() + return item +end + +function Base.IndexStyle(::Type{NonIntNestedLength}) + return IndexLinear() +end + +function Base.size(::NonIntNestedLength) + return (1,) +end + +function Base.length(::NonIntNestedLength) + return Int32(1) +end + +function Base.getindex(::NonIntNestedLength, ::Int) + return Int32(1) +end + +function deepnestedlist(depth::Int) + values = Int[1] + for _ in 1:depth + values = Parquet.ListVector(Int32[0, 1], values) + end + return values +end + +function deepnestedmap(depth::Int) + values = Int[1] + for _ in 1:depth + values = Parquet.MapVector(Int32[0, 1], Int[1], values) + end + return values +end + +@testset "Nested list vectors" begin + required = Parquet.ListVector(Int64[0, 2, 2, 3], Int32[10, 20, 30]) + @test required.offsets isa Vector{Int32} + @test eltype(required) === Parquet.ListValue{Int32} + @test fieldtype(typeof(required), :values) === AbstractVector{Int32} + @test size(required) == (3,) + @test collect(required[1]) == Int32[10, 20] + @test isempty(required[2]) + @test collect(required[3]) == Int32[30] + @test copy(required[1]) == Int32[10, 20] + @test_throws Base.CanonicalIndexError setindex!(required, required[1], 1) + @test_throws Base.CanonicalIndexError setindex!(required[1], Int32(1), 1) + + optional = Parquet.ListVector(Int32[0, 0, 0, 2], Int32[1, 2]; + validity=Bool[false, true, true]) + @test Missing <: eltype(optional) + @test ismissing(optional[1]) + @test !ismissing(optional[2]) && isempty(optional[2]) + @test collect(optional[3]) == Int32[1, 2] + + child = Parquet.ListVector(Int32[0, 1, 2], Int32[4, 5]) + outer = Parquet.ListVector(Int32[0, 2], child) + @test eltype(outer) === Parquet.ListValue{Parquet.ListValue{Int32}} + shallow = collect(outer[1]) + @test shallow isa Vector{eltype(child)} + @test shallow[1] == child[1] + @test shallow[2] == child[2] + + deepnestedlist(1) + deepresult = @timed deepnestedlist(128) + typerepr = @timed sprint(show, typeof(deepresult.value)) + @test ncodeunits(typerepr.value) < 16 * 1024 + @test count("ListVector", typerepr.value) == 1 + @test deepresult.bytes < 64 * 1024 * 1024 + @test deepresult.time < 5.0 + @test typerepr.bytes < 4 * 1024 * 1024 + @test typerepr.time < 2.0 + + @test_throws ArgumentError Parquet.ListVector(Int[1], Int[]) + @test_throws ArgumentError Parquet.ListVector(Int[0, 2, 1], Int[1]) + @test_throws ArgumentError Parquet.ListVector(Int[0, 2], Int[1]) + @test_throws ArgumentError Parquet.ListVector(Int[0, 0], Bool[], Int[]) + @test_throws ArgumentError Parquet.ListVector(Int[0, 1], Bool[false], Int[1]) + malformed = Parquet.ListValue(Int32[], 1, 1) + @test_throws ArgumentError malformed[1] + @test_throws ArgumentError Parquet.ListVector(Int32[0, 1], + NonIntNestedLength()) + owner = Ref{Any}(nothing) + backing = MutatingNestedVector(Int32[7], () -> begin + empty!(owner[].values) + return + end) + owner[] = backing + @test_throws ArgumentError Parquet.ListValue(backing, 1, 1)[1] + + if Sys.WORD_SIZE == 64 + large = Int(typemax(Int32)) + 1 + virtual = VirtualNestedVector{UInt8}(large) + wide = Parquet.ListVector(Int64[0, large], virtual) + @test wide.offsets isa Vector{Int64} + @test length(wide[1]) == large + end + @test Parquet._nestedindextype(typemax(Int32)) === Int32 + @test Parquet._nestedindextype(Int64(typemax(Int32)) + 1) === Int64 +end + +@testset "Nested struct vectors" begin + required = Parquet.StructVector(["id", "name"], + (Int32[1, 2], String["one", "two"])) + firstvalue = required[1] + @test eltype(required) <: Parquet.StructValue + @test firstvalue[1] == Int32(1) + @test firstvalue["name"] == "one" + @test firstvalue[:name] == "one" + @test collect(firstvalue) == Pair{String,Any}["id" => Int32(1), "name" => "one"] + @test copy(firstvalue) == collect(firstvalue) + @test NamedTuple(firstvalue) == (id=Int32(1), name="one") + @test_throws KeyError firstvalue["absent"] + @test_throws Base.CanonicalIndexError setindex!(required, firstvalue, 1) + + duplicate = Parquet.StructVector(["x", "x"], (Int[1], String["a"]))[1] + @test duplicate[1] == 1 + @test duplicate[2] == "a" + @test collect(duplicate) == Pair{String,Any}["x" => 1, "x" => "a"] + @test_throws ArgumentError duplicate["x"] + @test_throws ArgumentError NamedTuple(duplicate) + + optional = Parquet.StructVector(["id", "name"], + Int32[0, 0, 1, 2, 2], (Int32[7, 8], String["a", "b"])) + @test optional.ranks isa Vector{Int32} + @test size(optional) == (4,) + @test ismissing(optional[1]) + @test optional[2]["id"] == Int32(7) + @test optional[3]["name"] == "b" + @test ismissing(optional[4]) + + empty = Parquet.StructVector(String[], (); rows=2) + @test length(empty) == 2 + @test isempty(collect(empty[1])) + @test NamedTuple(empty[1]) == NamedTuple() + + nulname = Parquet.StructVector(["a\0b"], (Int[1],))[1] + @test nulname["a\0b"] == 1 + @test_throws ArgumentError NamedTuple(nulname) + + left = Parquet.StructVector(["a", "b"], + (Float64[NaN], Union{Missing,Int}[missing]))[1] + right = Parquet.StructVector(["a", "b"], + (Float64[NaN], Union{Missing,Int}[missing]))[1] + @test isequal(left, right) + @test hash(left) == hash(right) + @test (left == right) === false + + nested = Parquet.StructVector(["items"], (Parquet.ListVector( + Int[0, 1], Int[9]),)) + copied = copy(nested[1]) + @test copied[1].second isa Parquet.ListValue + @test collect(copied[1].second) == [9] + + width = 1_024 + widechildren = AbstractVector[Int[index] for index in 1:width] + wide = Parquet.StructVector(["field_$index" for index in 1:width], widechildren) + @test wide.children isa Vector{AbstractVector} + @test fieldtype(typeof(wide), :children) === Vector{AbstractVector} + @test typeof(wide[1]) === Parquet.StructValue + @test fieldtype(Parquet.StructValue, :children) === Vector{AbstractVector} + @test wide[1][width] == width + + @test_throws ArgumentError Parquet.StructVector(["a"], ()) + @test_throws ArgumentError Parquet.StructVector(String[], ()) + @test_throws ArgumentError Parquet.StructVector(String[], Int[0], ()) + @test_throws ArgumentError Parquet.StructVector(["a"], (Int[1],); rows=2) + @test_throws ArgumentError Parquet.StructVector(["a"], Int[1, 1], (Int[],)) + @test_throws ArgumentError Parquet.StructVector(["a"], Int[0, 2], (Int[1, 2],)) + @test_throws ArgumentError Parquet.StructVector(["a"], Int[0, 1], (Int[],)) + malformed = Parquet.StructValue(["x"], AbstractVector[Int32[]], 1) + @test_throws ArgumentError malformed[1] + sentinel = ErrorException("unrelated child metric was called") + selected = Parquet.StructValue(["x", "y"], + AbstractVector[Int32[7], ThrowNestedMetric(sentinel)], 1) + @test selected[1] == Int32(7) + owner = Ref{Any}(nothing) + child = MutatingNestedVector(Int32[7], () -> begin + empty!(owner[].values) + return + end) + owner[] = child + value = Parquet.StructValue(["x"], AbstractVector[child], 1) + @test_throws ArgumentError value[1] + @test_throws ArgumentError Parquet.StructVector(["x"], + AbstractVector[NonIntNestedLength()]) +end + +@testset "Nested map vectors" begin + column = Parquet.MapVector(Int32[0, 3, 3], Int32[1, 2, 1], + String["first", "middle", "last"]) + value = column[1] + @test eltype(column) === Parquet.MapValue{Int32,String,true} + @test fieldtype(typeof(column), :keys) === AbstractVector{Int32} + @test fieldtype(typeof(column), :values) === Union{Nothing,AbstractVector{String}} + @test collect(value) == Pair{Int32,String}[ + Int32(1) => "first", Int32(2) => "middle", Int32(1) => "last"] + @test value[1] == (Int32(1) => "first") + @test Parquet.maplookup(value, Int32(1)) == "last" + @test Parquet.maplookup(value, Int32(3), "default") == "default" + @test_throws KeyError Parquet.maplookup(value, Int32(3)) + @test Dict(value) == Dict(Int32(1) => "last", Int32(2) => "middle") + @test isempty(column[2]) + @test copy(value) == collect(value) + @test_throws Base.CanonicalIndexError setindex!(value, Int32(1) => "new", 1) + + deepnestedmap(1) + deepresult = @timed deepnestedmap(128) + typerepr = @timed sprint(show, typeof(deepresult.value)) + @test ncodeunits(typerepr.value) < 32 * 1024 + @test count("MapVector", typerepr.value) == 1 + @test deepresult.bytes < 96 * 1024 * 1024 + @test deepresult.time < 5.0 + @test typerepr.bytes < 4 * 1024 * 1024 + @test typerepr.time < 2.0 + + omitted = Parquet.MapVector(Int[0, 2], String["a", "b"]) + @test eltype(omitted) === Parquet.MapValue{String,Missing,false} + @test isequal(collect(omitted[1]), + Pair{String,Missing}["a" => missing, "b" => missing]) + @test isequal(Dict(omitted[1]), Dict("a" => missing, "b" => missing)) + + optional = Parquet.MapVector(Int[0, 0, 0, 1], Bool[false, true, true], + String["a"], Int[1]) + @test ismissing(optional[1]) + @test !ismissing(optional[2]) && isempty(optional[2]) + @test collect(optional[3]) == ["a" => 1] + + booleankeys = Bool[true, false] + booleanvalues = Int[10, 20] + booleans = Parquet.MapVector(Int[0, 1, 2], booleankeys, booleanvalues) + @test booleans.validity === nothing + @test booleans.keys === booleankeys + @test booleans.values === booleanvalues + @test collect(booleans[1]) == [true => 10] + @test collect(booleans[2]) == [false => 20] + + optionalkeyonly = Parquet.MapVector(Int[0, 0, 1], String["a"]; + validity=Bool[false, true]) + @test ismissing(optionalkeyonly[1]) + @test isequal(collect(optionalkeyonly[2]), Pair{String,Missing}["a" => missing]) + + bytekeys = [UInt8[1], UInt8[1]] + bytes = Parquet.MapVector(Int[0, 2], bytekeys, String["old", "new"])[1] + dictionary = Dict(bytes) + storedkey = only(keys(dictionary)) + @test storedkey !== bytekeys[1] + @test storedkey !== bytekeys[2] + bytekeys[1][1] = 2 + bytekeys[2][1] = 3 + @test dictionary[UInt8[1]] == "new" + + listkeycolumn = Parquet.ListVector(Int[0, 2], Int[1, 2]) + listkey = listkeycolumn[1] + listdictionary = Dict(Parquet.MapVector(Int[0, 1], [listkey], ["value"])[1]) + @test listdictionary[listkey] == "value" + storedlistkey = only(keys(listdictionary)) + listkeycolumn.values[1] = 9 + originallistkey = Parquet.ListVector(Int[0, 2], Int[1, 2])[1] + @test listdictionary[originallistkey] == "value" + @test !haskey(listdictionary, listkey) + @test isequal(storedlistkey, originallistkey) + @test hash(storedlistkey) == hash(originallistkey) + @test_throws Base.CanonicalIndexError setindex!(storedlistkey, 3, 1) + + structchild = Int[1] + structkey = Parquet.StructVector(["x"], (structchild,))[1] + structdictionary = Dict(Parquet.MapVector(Int[0, 1], [structkey], ["value"])[1]) + structchild[1] = 9 + originalstructkey = Parquet.StructVector(["x"], (Int[1],))[1] + @test structdictionary[originalstructkey] == "value" + @test !haskey(structdictionary, structkey) + + tuplechild = Int[1] + tuplekey = (Parquet.ListVector(Int[0, 1], tuplechild)[1], :tag) + tupledictionary = Dict(Parquet.MapVector(Int[0, 1], [tuplekey], ["value"])[1]) + tuplechild[1] = 9 + originaltuplekey = (Parquet.ListVector(Int[0, 1], Int[1])[1], :tag) + @test tupledictionary[originaltuplekey] == "value" + + namedchild = Int[1] + namedkey = (items=Parquet.ListVector(Int[0, 1], namedchild)[1], tag=:tag) + nameddictionary = Dict(Parquet.MapVector(Int[0, 1], [namedkey], ["value"])[1]) + namedchild[1] = 9 + originalnamedkey = (items=Parquet.ListVector(Int[0, 1], Int[1])[1], tag=:tag) + @test nameddictionary[originalnamedkey] == "value" + + equalkeycolumn = Parquet.ListVector(Int[0, 2, 4], Int[1, 2, 1, 2]) + firstkey = equalkeycolumn[1] + secondkey = equalkeycolumn[2] + @test isequal(firstkey, secondkey) + @test hash(firstkey) == hash(secondkey) + equaldictionary = Dict(Parquet.MapVector( + Int[0, 2], [firstkey, secondkey], ["old", "new"])[1]) + @test length(equaldictionary) == 1 + @test equaldictionary[firstkey] == "new" + + nullablekeycolumn = Parquet.ListVector(Int[0, 1], Union{Missing,Int}[missing]) + nullablekey = nullablekeycolumn[1] + @test Dict(Parquet.MapVector(Int[0, 1], [nullablekey], [1])[1])[nullablekey] == 1 + + bytechildren = [UInt8[1]] + bytechild = Parquet.ListVector(Int[0, 1], bytechildren)[1] + bytedictionary = Dict(Parquet.MapVector(Int[0, 1], [bytechild], [1])[1]) + bytechildren[1][1] = 2 + originalbytechild = Parquet.ListVector(Int[0, 1], [UInt8[1]])[1] + @test bytedictionary[originalbytechild] == 1 + + inner = Parquet.MapVector(Int[0, 1], Int[1], Int[2])[1] + @test_throws ArgumentError Dict(Parquet.MapVector(Int[0, 1], [inner], Int[3])[1]) + @test_throws ArgumentError Dict(Parquet.MapVector( + Int[0, 1], [UnsupportedNestedKey(Int[1])], Int[1])[1]) + @test_throws ArgumentError Dict(Parquet.MapVector( + Int[0, 1], [MutableNestedKey(1)], Int[1])[1]) + + @test_throws ArgumentError Parquet.MapVector(Int[0, 1], Union{Missing,Int}[1], Int[1]) + @test_throws ArgumentError Parquet.MapVector(Int[0, 1], Any[missing], Int[1]) + @test_throws ArgumentError Parquet.MapVector(Int[0, 1], Int[1], Int[]) + @test_throws ArgumentError Parquet.MapVector(Int[0, 2], Int[1], Int[1]) + @test_throws ArgumentError Parquet.MapVector(Int[0, 1], Bool[false], Int[1], Int[1]) + @test_throws ArgumentError Parquet.MapVector(Int32[0, 1], + NonIntNestedLength(), Int32[2]) + @test_throws ArgumentError Parquet.MapVector(Int32[0, 1], Int32[1], + NonIntNestedLength()) + + attackkeys = Int32[1] + values = MutatingNestedVector(Int32[7], () -> begin + empty!(attackkeys) + return + end) + view = Parquet.MapValue{Int32,Int32,true}(attackkeys, values, 1, 1) + @test_throws ArgumentError view[1] + + owner = Ref{Any}(nothing) + attackkeys = MutatingNestedVector(Int32[1], () -> begin + empty!(owner[].values) + return + end) + owner[] = attackkeys + view = Parquet.MapValue{Int32,Missing,false}(attackkeys, nothing, 1, 1) + @test_throws ArgumentError view[1] +end diff --git a/test/page.jl b/test/page.jl new file mode 100644 index 0000000..b2b4d4c --- /dev/null +++ b/test/page.jl @@ -0,0 +1,564 @@ +using Random + +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end +if !isdefined(Parquet, :readpage) + Base.include(Parquet, joinpath(@__DIR__, "..", "src", "page.jl")) +end + +const PAGE_CORPUS = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + +function pagecorpus(parts...) + return joinpath(PAGE_CORPUS, "data", parts...) +end + +function pagev1header(count::Integer; encoding=MD.Encoding.PLAIN, levelencoding=MD.Encoding.RLE) + return MD.DataPageHeader(num_values=Int32(count), encoding=encoding, definition_level_encoding=levelencoding, + repetition_level_encoding=MD.Encoding.RLE) +end + +function pagev2header(count::Integer; nulls=0, rows=count, definition=0, repetition=0, + encoding=MD.Encoding.PLAIN, is_compressed=nothing) + return MD.DataPageHeaderV2(num_values=Int32(count), num_nulls=Int32(nulls), + num_rows=Int32(rows), encoding=encoding, + definition_levels_byte_length=Int32(definition), + repetition_levels_byte_length=Int32(repetition), is_compressed=is_compressed) +end + +# Serialize a page: Thrift header followed by the payload. `crc` is :valid, :none, or an Int32. +function pagebytes(payload::Vector{UInt8}; type=MD.PageType.DATA_PAGE, v1=nothing, + index=nothing, dict=nothing, v2=nothing, + crc=:valid, compressed=length(payload), uncompressed=compressed) + crcvalue = crc === :valid ? reinterpret(Int32, Parquet.pagechecksum(payload)) : crc === :none ? nothing : Int32(crc) + header = MD.PageHeader(type_=type, uncompressed_page_size=Int32(uncompressed), compressed_page_size=Int32(compressed), + crc=crcvalue, data_page_header=v1, index_page_header=index, + dictionary_page_header=dict, data_page_header_v2=v2) + return vcat(TH.encode(header), payload) +end + +function plainpage(values::Vector{Int32}; kwargs...) + return pagebytes(Parquet.encode_plain(values); v1=pagev1header(length(values)), kwargs...) +end + +function readframe(bytes::Vector{UInt8}; offset=0, stop=length(bytes), limits=Parquet.Limits()) + src = Parquet.source(bytes) + return Parquet.readpage(src, Int64(offset), Int64(stop), limits) +end + +mutable struct PageBoundSource <: Parquet.AbstractSource + bytes::Vector{UInt8} + reads::Int +end + +function Parquet.sourcelength(source::PageBoundSource) + return Int64(length(source.bytes)) +end + +function Parquet.readrange(source::PageBoundSource, offset::Integer, + count::Integer) + source.reads += 1 + first = Int(offset) + 1 + return @view source.bytes[first:(first + Int(count) - 1)] +end + +struct ShiftedPageBytes <: AbstractVector{UInt8} + bytes::Vector{UInt8} +end + +function Base.IndexStyle(::Type{ShiftedPageBytes}) + return IndexLinear() +end + +function Base.size(bytes::ShiftedPageBytes) + return (length(bytes.bytes),) +end + +function Base.axes(bytes::ShiftedPageBytes) + return (2:(length(bytes.bytes) + 1),) +end + +function Base.getindex(bytes::ShiftedPageBytes, index::Int) + checkbounds(bytes, index) + return bytes.bytes[index - 1] +end + +mutable struct PageCallbackSentinel <: Exception + id::Int +end + +mutable struct PageContractSource <: Parquet.AbstractSource + bytes::Vector{UInt8} + mode::Symbol + faultread::Int + lengthcalls::Int + reads::Vector{Tuple{Int64,Int64}} + sentinel::Union{Nothing,PageCallbackSentinel} +end + +function PageContractSource(bytes::Vector{UInt8}; mode::Symbol=:normal, + faultread::Int=0, sentinel=nothing) + return PageContractSource(bytes, mode, faultread, 0, + Tuple{Int64,Int64}[], sentinel) +end + +function Parquet.sourcelength(source::PageContractSource) + source.lengthcalls += 1 + source.mode === :length_throw && throw(something(source.sentinel)) + source.mode === :length_type && return Float64(length(source.bytes)) + source.mode === :length_negative && return Int64(-1) + source.mode === :length_changing && source.lengthcalls > 1 && return Int64(0) + return Int64(length(source.bytes)) +end + +function Parquet.readrange(source::PageContractSource, offset::Integer, + count::Integer) + offset64 = Int64(offset) + count64 = Int64(count) + push!(source.reads, (offset64, count64)) + if length(source.reads) == source.faultread + source.mode === :throw && throw(something(source.sentinel)) + source.mode === :short && return fill(UInt8(0), max(Int(count64) - 1, 0)) + source.mode === :long && return fill(UInt8(0), Int(count64) + 1) + source.mode === :wrong_type && return fill(Int8(0), Int(count64)) + source.mode === :wrong_axes && return ShiftedPageBytes( + fill(UInt8(0), Int(count64))) + end + first = Int(offset64) + 1 + return @view source.bytes[first:(first + Int(count64) - 1)] +end + +@testset "page header parsing and bounds" begin + page = plainpage(Int32[1, 2, 3]) + src = Parquet.source(page) + header, headerlength = Parquet.readpageheader(src, Int64(0), Int64(length(page)), Parquet.Limits()) + @test header.type_ == MD.PageType.DATA_PAGE && header.compressed_page_size == 12 + @test headerlength == length(page) - 12 + @test header.data_page_header.num_values == 3 && header.crc !== nothing + @test Parquet.validatepageheader(header) === :data_v1 + sharedbudget = Parquet._LiveByteBudget(Parquet.Limits()) + sharedheader, sharedlength = Parquet.readpageheader(src, Int64(0), + Int64(length(page)), Parquet.Limits(); budget=sharedbudget) + @test sharedheader == header + @test sharedlength == headerlength + retainedcharge = Parquet._budgetused(sharedbudget) + @test retainedcharge > 0 + Parquet._release!(sharedbudget, retainedcharge) + frame = readframe(page) + @test frame.offset == 0 && frame.headerlength == headerlength && Parquet.pageend(frame) == length(page) + @test collect(frame.payload) == Parquet.encode_plain(Int32[1, 2, 3]) && Parquet.pagekind(frame) === :data_v1 + padded = vcat(UInt8[0xaa, 0xbb], page, UInt8[0xcc]) + shifted = readframe(padded; offset=2, stop=2 + length(page)) + @test shifted.offset == 2 && Parquet.pageend(shifted) == 2 + length(page) + @test collect(shifted.payload) == collect(frame.payload) + @test_throws Parquet.FormatError readframe(page; offset=length(page)) + @test_throws Parquet.FormatError readframe(page; offset=-1) + @test_throws Parquet.FormatError readframe(page; stop=headerlength - 1) + @test_throws Parquet.FormatError readframe(page; stop=headerlength + 5) + @test_throws Parquet.FormatError readframe(page[1:(end - 1)]) + @test_throws Parquet.FormatError Parquet.readpageheader(src, Int64(0), Int64(headerlength - 1), Parquet.Limits()) + outofbounds = PageBoundSource(page, 0) + zerolimit = Parquet.Limits(max_page_header_bytes=0) + headererror = try + Parquet.readpageheader(outofbounds, Int64(0), + Int64(length(page) + 1), zerolimit) + nothing + catch err + err + end + @test headererror isa Parquet.FormatError + @test headererror.message == + "page read stop $(length(page) + 1) is past the end of the source" + @test outofbounds.reads == 0 + pageerror = try + Parquet.readpage(outofbounds, Int64(0), + Int64(length(page) + 1), zerolimit) + nothing + catch err + err + end + @test pageerror isa Parquet.FormatError + @test pageerror.message == headererror.message + @test outofbounds.reads == 0 + for n in 0:(headerlength - 1) + @test_throws Parquet.FormatError readframe(page[1:n]) + end + truncatedheader = page[1:(headerlength - 1)] + @test_throws Parquet.FormatError readframe(truncatedheader; + limits=Parquet.Limits(max_page_header_bytes=length(truncatedheader))) + tight = Parquet.Limits(max_page_header_bytes=headerlength) + @test Parquet.readpage(src, Int64(0), Int64(length(page)), tight).headerlength == headerlength + @test_throws Parquet.LimitError readframe(page; limits=Parquet.Limits(max_page_header_bytes=headerlength - 1)) + @test_throws Parquet.LimitError readframe(page; limits=Parquet.Limits(max_page_header_bytes=1)) + @test_throws Parquet.LimitError readframe(page; limits=Parquet.Limits(max_page_header_bytes=0)) + @test_throws Parquet.LimitError readframe(page; limits=Parquet.Limits(max_page_bytes=11)) + @test readframe(page; limits=Parquet.Limits(max_page_bytes=12)).header.compressed_page_size == 12 + bigger = pagebytes(Parquet.encode_plain(Int32[1, 2, 3]); v1=pagev1header(3), uncompressed=4096) + @test_throws Parquet.LimitError readframe(bigger; limits=Parquet.Limits(max_page_bytes=4095)) + @test_throws Parquet.FormatError readframe(pagebytes(UInt8[]; v1=pagev1header(0), compressed=-1, uncompressed=0)) + @test_throws Parquet.FormatError readframe(pagebytes(UInt8[]; v1=pagev1header(0), compressed=0, uncompressed=-1)) + @test_throws Parquet.FormatError readframe(pagebytes(UInt8[0x01]; + v1=pagev1header(0), compressed=typemax(Int32))) + truncated = pagebytes(UInt8[0x01]; v1=pagev1header(0), compressed=2) + @test_throws Parquet.FormatError readframe(truncated) + @test_throws Parquet.FormatError readframe(truncated; + limits=Parquet.Limits(max_page_bytes=1)) +end + +@testset "page exact source reads" begin + page = plainpage(Int32[1, 2, 3]) + changing = PageContractSource(page; mode=:length_changing) + frame = Parquet.readpage(changing, Int64(0), Int64(length(page)), + Parquet.Limits()) + @test collect(frame.payload) == Parquet.encode_plain(Int32[1, 2, 3]) + @test changing.lengthcalls == 1 + @test length(changing.reads) == 2 + + for mode in (:short, :long, :wrong_type, :wrong_axes) + for faultread in 1:2 + malformed = PageContractSource(page; mode=mode, + faultread=faultread) + budget = Parquet._LiveByteBudget(Parquet.Limits()) + @test_throws ArgumentError Parquet.readpage(malformed, Int64(0), + Int64(length(page)), Parquet.Limits(); budget=budget) + @test length(malformed.reads) == faultread + @test Parquet._budgetused(budget) == 0 + end + end + + for faultread in 1:2 + sentinel = PageCallbackSentinel(faultread) + throwing = PageContractSource(page; mode=:throw, + faultread=faultread, sentinel=sentinel) + budget = Parquet._LiveByteBudget(Parquet.Limits()) + error = try + Parquet.readpage(throwing, Int64(0), Int64(length(page)), + Parquet.Limits(); budget=budget) + nothing + catch err + err + end + @test error === sentinel + @test length(throwing.reads) == faultread + @test Parquet._budgetused(budget) == 0 + end + + for mode in (:length_type, :length_negative) + malformed = PageContractSource(page; mode=mode) + @test_throws ArgumentError Parquet.readpage(malformed, Int64(0), + Int64(length(page)), Parquet.Limits()) + @test malformed.lengthcalls == 1 + @test isempty(malformed.reads) + end + sentinel = PageCallbackSentinel(3) + throwinglength = PageContractSource(page; mode=:length_throw, + sentinel=sentinel) + error = try + Parquet.readpage(throwinglength, Int64(0), Int64(length(page)), + Parquet.Limits()) + nothing + catch err + err + end + @test error === sentinel + @test isempty(throwinglength.reads) +end + +@testset "offset-index exact source reads" begin + bytes = TH.encode(MD.OffsetIndex(page_locations=MD.PageLocation[])) + normal = PageContractSource(bytes; mode=:length_changing) + footer = Parquet.Footer(Int64(length(bytes)), Int64(0), false, UInt8[]) + file = Parquet.File(normal, footer, false) + index = Parquet._decodeoffsetindex(file, Int64(0), Int64(length(bytes)), + Parquet.Limits(), Parquet._LiveByteBudget(Parquet.Limits())) + @test isempty(index.page_locations) + @test normal.lengthcalls == 1 + @test normal.reads == [(Int64(0), Int64(length(bytes)))] + + for mode in (:short, :long, :wrong_type, :wrong_axes) + malformed = PageContractSource(bytes; mode=mode, faultread=1) + malformedfile = Parquet.File(malformed, footer, false) + budget = Parquet._LiveByteBudget(Parquet.Limits()) + @test_throws ArgumentError Parquet._decodeoffsetindex(malformedfile, + Int64(0), Int64(length(bytes)), Parquet.Limits(), budget) + @test malformed.lengthcalls == 1 + @test length(malformed.reads) == 1 + @test Parquet._budgetused(budget) == 0 + end + + sentinel = PageCallbackSentinel(4) + throwing = PageContractSource(bytes; mode=:throw, faultread=1, + sentinel=sentinel) + throwingfile = Parquet.File(throwing, footer, false) + budget = Parquet._LiveByteBudget(Parquet.Limits()) + error = try + Parquet._decodeoffsetindex(throwingfile, Int64(0), + Int64(length(bytes)), Parquet.Limits(), budget) + nothing + catch err + err + end + @test error === sentinel + @test Parquet._budgetused(budget) == 0 +end + +@testset "checked page frame count and arithmetic" begin + exactlimits = Parquet.Limits(max_container_elements=1) + @test Parquet._nextpageframecount(Int64(0), exactlimits) == 1 + limiterror = try + Parquet._nextpageframecount(Int64(1), exactlimits) + nothing + catch err + err + end + @test limiterror isa Parquet.LimitError + @test limiterror.resource == :container_elements + @test limiterror.requested == 2 + @test limiterror.maximum == 1 + @test_throws ArgumentError Parquet._nextpageframecount(Int64(-1), + exactlimits) + overflow = try + Parquet._nextpageframecount(typemax(Int64), Parquet.Limits( + max_container_elements=typemax(Int64))) + nothing + catch err + err + end + @test overflow isa Parquet.LimitError + @test overflow.resource == :container_elements + @test overflow.requested == typemax(Int64) + + frame = readframe(plainpage(Int32[1])) + compressed = Int64(frame.header.compressed_page_size) + offset = typemax(Int64) - Int64(frame.headerlength) - compressed + exact = Parquet.PageFrame(offset, frame.header, frame.headerlength, + frame.payload, frame.materializedcharge) + @test Parquet.pageend(exact) == typemax(Int64) + beyond = Parquet.PageFrame(offset + 1, frame.header, + frame.headerlength, frame.payload, frame.materializedcharge) + frameerror = try + Parquet.pageend(beyond) + nothing + catch err + err + end + @test frameerror isa Parquet.FormatError + @test frameerror.message == "page frame end overflows Int64" + @test_throws Parquet.FormatError Parquet._pageframeend(Int64(-1), 1, 0) + @test_throws Parquet.FormatError Parquet._pageframeend(Int64(0), -1, 0) + @test_throws Parquet.FormatError Parquet._pageframeend(Int64(0), + typemax(UInt128), 0) +end + +@testset "page header materialization preflight" begin + payload = fill(UInt8(0x61), 100_000) + statistics = MD.Statistics(max=payload) + data = MD.DataPageHeader(num_values=Int32(0), encoding=MD.Encoding.PLAIN, + definition_level_encoding=MD.Encoding.RLE, + repetition_level_encoding=MD.Encoding.RLE, statistics=statistics) + header = MD.PageHeader(type_=MD.PageType.DATA_PAGE, + uncompressed_page_size=Int32(0), compressed_page_size=Int32(0), + data_page_header=data) + bytes = TH.encode(header) + src = Parquet.source(bytes) + limits = Parquet.Limits(max_materialized_bytes=700, + max_page_header_bytes=200_000) + function rejectlargestatistics() + budget = Parquet._LiveByteBudget(limits) + @test_throws Parquet.LimitError Parquet.readpage(src, Int64(0), + Int64(length(bytes)), limits; budget=budget) + @test Parquet._budgetused(budget) == 0 + return + end + rejectlargestatistics() + GC.gc() + @test @allocated(rejectlargestatistics()) < 10_000 +end + +@testset "page CRC32" begin + payload = collect(codeunits("123456789")) + @test Parquet.pagechecksum(payload) == 0xcbf43926 + page = pagebytes(payload; v1=pagev1header(0)) + frame = readframe(page) + @test frame.header.crc == reinterpret(Int32, 0xcbf43926) && frame.header.crc < 0 + @test collect(frame.payload) == payload + corrupted = copy(page) + corrupted[end] ⊻= 0x01 + @test_throws Parquet.FormatError readframe(corrupted) + wrongcrc = pagebytes(payload; v1=pagev1header(0), crc=Int32(0)) + @test_throws Parquet.FormatError readframe(wrongcrc) + nocrc = pagebytes(payload; v1=pagev1header(0), crc=:none) + @test readframe(nocrc).header.crc === nothing + damaged = copy(nocrc) + damaged[end] ⊻= 0x01 + @test collect(readframe(damaged).payload) != payload + empty = pagebytes(UInt8[]; v1=pagev1header(0)) + @test readframe(empty).header.crc == 0 && isempty(readframe(empty).payload) + + src = Parquet.source(page) + probe = Parquet._LiveByteBudget(Parquet.Limits()) + probeframe = Parquet.readpage(src, Int64(0), Int64(length(page)), + Parquet.Limits(); budget=probe) + retained = Parquet._budgetused(probe) + scratch = Parquet._pagechecksumscratch(probeframe.payload) + Parquet._release!(probe, probeframe.materializedcharge) + constrainedlimits = Parquet.Limits( + max_materialized_bytes=retained + scratch - 1) + constrained = Parquet._LiveByteBudget(constrainedlimits) + @test_throws Parquet.LimitError Parquet.readpage(src, Int64(0), + Int64(length(page)), constrainedlimits; budget=constrained) + @test Parquet._budgetused(constrained) == 0 + exactlimits = Parquet.Limits(max_materialized_bytes=retained + scratch) + exact = Parquet._LiveByteBudget(exactlimits) + exactframe = Parquet.readpage(src, Int64(0), Int64(length(page)), + exactlimits; budget=exact) + @test Parquet._budgetused(exact) == exactframe.materializedcharge == retained + Parquet._release!(exact, exactframe.materializedcharge) + @test Parquet._budgetused(exact) == 0 + Parquet.close!(src) +end + +@testset "page type and header consistency" begin + payload = Parquet.encode_plain(Int32[7]) + v1 = pagev1header(1) + index = MD.IndexPageHeader() + dict = MD.DictionaryPageHeader(num_values=Int32(1), encoding=MD.Encoding.PLAIN) + v2 = pagev2header(1) + @test Parquet.pagekind(readframe(pagebytes(payload; type=MD.PageType.DICTIONARY_PAGE, dict=dict))) === :dictionary + @test Parquet.pagekind(readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, v2=v2))) === :data_v2 + @test Parquet.pagekind(readframe(pagebytes(payload; type=MD.PageType.INDEX_PAGE, + index=index))) === :index + unknown = readframe(pagebytes(payload; type=MD.PageType.T(9))) + @test Parquet.pagekind(unknown) === :unknown && Parquet.pageend(unknown) == length(pagebytes(payload; type=MD.PageType.T(9))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE, dict=dict)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE, v1=v1, v2=v2)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE, v1=v1, dict=dict)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; + type=MD.PageType.DATA_PAGE, v1=v1, index=index)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; + type=MD.PageType.INDEX_PAGE)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; + type=MD.PageType.INDEX_PAGE, index=index, v1=v1)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; + type=MD.PageType.INDEX_PAGE, index=index, dict=dict)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; + type=MD.PageType.INDEX_PAGE, index=index, v2=v2)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DICTIONARY_PAGE, v1=v1)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DICTIONARY_PAGE)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; + type=MD.PageType.DICTIONARY_PAGE, dict=dict, index=index)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, v1=v1)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, v2=v2, dict=dict)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; + type=MD.PageType.DATA_PAGE_V2, v2=v2, index=index)) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(-1))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; nulls=-1))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; nulls=2))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; rows=-1))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; rows=2))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; definition=-1))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; repetition=-1))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; definition=length(payload) + 1))) + @test_throws Parquet.FormatError readframe(pagebytes(payload; type=MD.PageType.DATA_PAGE_V2, + v2=pagev2header(1; repetition=length(payload) + 1))) + oversized = pagebytes(payload; type=MD.PageType.T(9), compressed=length(payload) + 1) + @test_throws Parquet.FormatError readframe(oversized) +end + +@testset "page decompression gate" begin + payload = Parquet.encode_plain(Int32[1, 2]) + frame = readframe(pagebytes(payload; v1=pagev1header(2))) + @test collect(Parquet.decompresspage(frame, MD.CompressionCodec.UNCOMPRESSED)) == payload + @test_throws Parquet.FormatError Parquet.decompresspage(frame, MD.CompressionCodec.SNAPPY) + @test_throws Parquet.FormatError Parquet.decompresspage(frame, MD.CompressionCodec.ZSTD) + @test_throws Parquet.FormatError Parquet.decompresspage(frame, MD.CompressionCodec.T(99)) + mismatch = readframe(pagebytes(payload; v1=pagev1header(2), uncompressed=length(payload) + 1)) + @test_throws Parquet.FormatError Parquet.decompresspage(mismatch, MD.CompressionCodec.UNCOMPRESSED) +end + +@testset "page mutation fuzz" begin + page = plainpage(Int32[1, 2, 3, 4]) + rng = MersenneTwister(2026) + outcomes = Set{Symbol}() + for trial in 1:600 + mutated = copy(page) + for _ in 1:rand(rng, 1:3) + mutated[rand(rng, eachindex(mutated))] = rand(rng, UInt8) + end + result = try + readframe(mutated) + :ok + catch err + err + end + if result === :ok + push!(outcomes, :ok) + else + @test result isa Union{Parquet.FormatError,Parquet.LimitError} + push!(outcomes, nameof(typeof(result))) + end + end + @test :FormatError in outcomes + for n in 0:(length(page) - 1) + @test_throws Parquet.FormatError readframe(page[1:n]) + end +end + +# Page summaries of one column chunk: (kind, num_values, crc present, crc matches, sizes equal) or the error. +function corpuspages(path::String, column::Int) + file = Parquet.File(path) + meta = TH.decode(file.footer.bytes, MD.FileMetaData) + md = meta.row_groups[1].columns[column].meta_data + start = Int64(md.data_page_offset) + stop = start + Int64(md.total_compressed_size) + pages = [] + position = start + while position < stop + frame = try + Parquet.readpage(file.source, position, stop, Parquet.Limits()) + catch err + err + end + frame isa Exception && (push!(pages, frame); break) + header = frame.header + push!(pages, (Parquet.pagekind(frame), Int(header.data_page_header.num_values), header.crc !== nothing, + header.crc !== nothing && Parquet.pagechecksum(frame.payload) == reinterpret(UInt32, header.crc), + header.compressed_page_size == header.uncompressed_page_size)) + position = Parquet.pageend(frame) + end + close(file) + return pages +end + +@testset "official checksum fixtures" begin + if !isdir(pagecorpus()) + @warn "parquet-testing corpus not found; skipping page corpus tests" PAGE_CORPUS + else + for column in 1:2 + pages = corpuspages(pagecorpus("datapage_v1-uncompressed-checksum.parquet"), column) + @test length(pages) == 2 && all(page -> page isa Tuple, pages) + @test all(page -> page[1] === :data_v1 && page[3] && page[4] && page[5], pages) + @test sum(page -> page[2], pages) == 5120 + end + # README: column a has a bad CRC on page 0, column b on page 1 + first = corpuspages(pagecorpus("datapage_v1-corrupt-checksum.parquet"), 1) + @test length(first) == 1 && first[1] isa Parquet.FormatError + second = corpuspages(pagecorpus("datapage_v1-corrupt-checksum.parquet"), 2) + @test length(second) == 2 && second[1] isa Tuple && second[1][4] && second[2] isa Parquet.FormatError + tiny = corpuspages(pagecorpus("alltypes_tiny_pages_plain.parquet"), 1) + @test length(tiny) == 325 && all(page -> page isa Tuple && page[1] === :data_v1 && !page[3], tiny) + @test sum(page -> page[2], tiny) == 7300 + end +end diff --git a/test/plain.jl b/test/plain.jl new file mode 100644 index 0000000..264a251 --- /dev/null +++ b/test/plain.jl @@ -0,0 +1,50 @@ +@testset "PLAIN primitives" begin + for values in ( + Int32[typemin(Int32), -1, 0, 1, typemax(Int32)], + Int64[typemin(Int64), -1, 0, 1, typemax(Int64)], + Float32[-Inf, -0.0, 0.0, 1.5, Inf, NaN], + Float64[-Inf, -0.0, 0.0, 1.5, Inf, NaN], + ) + encoded = Parquet.encode_plain(values) + decoded, position = Parquet.decode_plain(eltype(values), encoded, length(values)) + @test isequal(decoded, values) + @test position == length(encoded) + 1 + @test_throws Parquet.FormatError Parquet.decode_plain(eltype(values), encoded[1:(end - 1)], length(values)) + end + + booleans = Bool[true, false, true, true, false, false, false, true, true] + encoded = Parquet.encode_plain(booleans) + @test encoded == UInt8[0x8d, 0x01] + decoded, position = Parquet.decode_plain(Bool, encoded, length(booleans)) + @test decoded == booleans + @test position == 3 + @test_throws Parquet.LimitError Parquet.decode_plain(Bool, encoded, 3; + limits=Parquet.Limits(max_container_elements=2)) + @test_throws Parquet.LimitError Parquet.decode_plain(Int32, zeros(UInt8, 12), 3; + limits=Parquet.Limits(max_container_elements=2)) +end + +@testset "PLAIN byte arrays" begin + values = [UInt8[], UInt8[0x00, 0xff], collect(codeunits("Parquet"))] + encoded = Parquet.encode_plain_byte_array(values) + decoded, position = Parquet.decode_plain_byte_array(encoded, length(values)) + @test decoded == values + @test position == length(encoded) + 1 + @test Parquet.encode_plain_byte_array(["abc"]) == UInt8[0x03, 0x00, 0x00, 0x00, 0x61, 0x62, 0x63] + + negative = UInt8[0xff, 0xff, 0xff, 0xff] + @test_throws Parquet.FormatError Parquet.decode_plain_byte_array(negative, 1) + oversized = UInt8[0x04, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04] + @test_throws Parquet.LimitError Parquet.decode_plain_byte_array(oversized, 1; + limits=Parquet.Limits(max_string_bytes=3)) + @test_throws Parquet.FormatError Parquet.decode_plain_byte_array(UInt8[0x00], 1000) +end + +@testset "PLAIN fixed byte arrays" begin + values = reshape(UInt8[0x01, 0x02, 0x03, 0x04, 0x05, 0x06], 2, 3) + encoded = Parquet.encode_plain_fixed(values) + decoded, position = Parquet.decode_plain_fixed(encoded, 3, 2) + @test decoded == values + @test position == 7 + @test_throws Parquet.FormatError Parquet.decode_plain_fixed(encoded, 4, 2) +end diff --git a/test/rle.jl b/test/rle.jl new file mode 100644 index 0000000..312bccd --- /dev/null +++ b/test/rle.jl @@ -0,0 +1,56 @@ +@testset "RLE and bit-packed hybrid" begin + values = UInt64[0, 1, 2, 3, 4, 5, 6, 7] + encoded = Parquet.encode_hybrid(values, 3) + @test encoded == UInt8[0x03, 0x88, 0xc6, 0xfa] + decoded, position = Parquet.decode_hybrid(encoded, 8, 3) + @test decoded == values + @test position == 5 + + rle = UInt8[0x10, 0x03] + decoded, position = Parquet.decode_hybrid(rle, 8, 3) + @test decoded == fill(UInt64(3), 8) + @test position == 3 + + prefixed = Parquet.encode_hybrid(UInt64[1, 0, 1], 1; length_prefix=true) + decoded, position = Parquet.decode_hybrid(prefixed, 3, 1; length_prefix=true) + @test decoded == UInt64[1, 0, 1] + @test position == length(prefixed) + 1 + + zerosencoded = Parquet.encode_hybrid(zeros(UInt64, 11), 0) + decoded, _ = Parquet.decode_hybrid(zerosencoded, 11, 0) + @test decoded == zeros(UInt64, 11) + + @test_throws Parquet.FormatError Parquet.decode_hybrid(UInt8[0x00], 1, 1) + @test_throws Parquet.FormatError Parquet.decode_hybrid(encoded[1:(end - 1)], 8, 3) + @test_throws Parquet.FormatError Parquet.decode_hybrid(fill(UInt8(0x80), 10), 1, 1) + hugeheader = vcat(fill(UInt8(0xff), 9), UInt8[0x01]) + @test_throws Parquet.FormatError Parquet.decode_hybrid(hugeheader, 1, 64) + hugerle = UInt8[] + Parquet._writehybridvarint!(hugerle, UInt64(typemax(Int32) + Int64(1)) << 1) + push!(hugerle, 0x00) + @test_throws Parquet.FormatError Parquet.decode_hybrid(hugerle, 1, 1) + hugegroups = UInt64(typemax(Int32) ÷ 8 + 1) + hugepacked = UInt8[] + Parquet._writehybridvarint!(hugepacked, (hugegroups << 1) | 0x01) + @test_throws Parquet.FormatError Parquet.decode_hybrid(hugepacked, 1, 1) + @test_throws ArgumentError Parquet.encode_hybrid(UInt64[8], 3) + @test_throws ArgumentError Parquet.decode_hybrid(encoded, 8, 65) +end + +@testset "deprecated BIT_PACKED levels" begin + bytes = UInt8[0x05, 0x39, 0x77] + values, position = Parquet.decode_bit_packed(bytes, 8, 3) + @test values == UInt64.(0:7) + @test position == 4 + padded = UInt8[0xaa, 0x05, 0x39, 0x70, 0xbb] + values, position = Parquet.decode_bit_packed(padded, 7, 3; offset=2) + @test values == UInt64.(0:6) + @test position == 5 + @test Parquet.decode_bit_packed(UInt8[], 4, 0) == (zeros(UInt64, 4), 1) + @test_throws Parquet.FormatError Parquet.decode_bit_packed(bytes[1:2], 8, 3) + @test_throws Parquet.FormatError Parquet.decode_bit_packed(UInt8[0x05, 0x39, 0x71], 7, 3) + @test_throws Parquet.LimitError Parquet.decode_bit_packed(bytes, 8, 3; + limits=Parquet.Limits(max_container_elements=7)) + @test_throws ArgumentError Parquet.decode_bit_packed(bytes, -1, 3) + @test_throws ArgumentError Parquet.decode_bit_packed(bytes, 8, 65) +end diff --git a/test/runtests.jl b/test/runtests.jl index f789d33..6e4fed4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,14 +1,46 @@ using Parquet using Test -using LazyArtifacts, Artifacts -# Note: readdir(...; join=true) requires Julia v1.4. -const parcompat = joinpath(artifact"parcompat", readdir(artifact"parcompat")[1]) -const julia_parcompat = joinpath(artifact"julia_parcompat", readdir(artifact"julia_parcompat")[1]) - -@testset "parquet tests" begin - include("test_load.jl") - include("test_codec.jl") - include("test_cursors.jl") - include("test_writer.jl") -end +include("api.jl") +include("limits.jl") +include("nested_vectors.jl") +include("source.jl") +include("footer.jl") +include("plain.jl") +include("rle.jl") +include("checksum.jl") +include("thrift.jl") +include("generator.jl") +include("metadata.jl") +include("schema.jl") +include("nested_schema.jl") +include("logical.jl") +include("logical_temporal.jl") +include("logical_json.jl") +include("logical_bson.jl") +include("logical_binary.jl") +include("logical_decimal.jl") +include("statistics.jl") +include("logical_corpus.jl") +include("delta.jl") +include("bss.jl") +include("page.jl") +include("codecs.jl") +include("column.jl") +include("nested_reader.jl") +include("nested_table.jl") +include("write.jl") +include("write_statistics.jl") +include("write_encoding_paths.jl") +include("write_logical.jl") +include("logical_column.jl") +include("write_nested.jl") +include("write_provenance.jl") +include("write_splitting.jl") +include("write_offset_index.jl") +include("dictionary.jl") +include("table.jl") +include("conformance/n5/model/runtests.jl") +include("conformance/n5/hardening/source_mutation.jl") +include("conformance/n5/external.jl") +include("conformance/n6/julia/runtests.jl") diff --git a/test/schema.jl b/test/schema.jl new file mode 100644 index 0000000..3f22c05 --- /dev/null +++ b/test/schema.jl @@ -0,0 +1,212 @@ +if !@isdefined(TH) + const TH = Parquet.Thrift +end +if !@isdefined(MD) + const MD = Parquet.Metadata +end + +function schemaelement(name; type=nothing, repetition=nothing, children=nothing, length=nothing) + return MD.SchemaElement(name=name, type_=type, repetition_type=repetition, + num_children=children, type_length=length) +end + +@testset "schema tree and levels" begin + elements = [ + schemaelement("root"; children=Int32(3)), + schemaelement("id"; type=MD.Type.INT64, repetition=MD.FieldRepetitionType.REQUIRED), + schemaelement("items"; repetition=MD.FieldRepetitionType.OPTIONAL, children=Int32(1)), + schemaelement("list"; repetition=MD.FieldRepetitionType.REPEATED, children=Int32(1)), + schemaelement("element"; type=MD.Type.BYTE_ARRAY, repetition=MD.FieldRepetitionType.OPTIONAL), + schemaelement("pair"; repetition=MD.FieldRepetitionType.REPEATED, children=Int32(2)), + schemaelement("key"; type=MD.Type.BYTE_ARRAY, repetition=MD.FieldRepetitionType.REQUIRED), + schemaelement("value"; type=MD.Type.INT32, repetition=MD.FieldRepetitionType.OPTIONAL), + ] + schema = Parquet.Schema(elements) + @test length(schema.root.children) == 3 + @test length(schema.leaves) == 4 + @test [leaf.column_index for leaf in schema.leaves] == Int32[1, 2, 3, 4] + @test schema.leaves[1].path == ["id"] + @test schema.leaves[2].path == ["items", "list", "element"] + @test schema.leaves[2].max_definition_level == 3 + @test schema.leaves[2].max_repetition_level == 1 + @test schema.leaves[3].path == ["pair", "key"] + @test schema.leaves[3].max_definition_level == 1 + @test schema.leaves[3].max_repetition_level == 1 + @test schema.leaves[4].max_definition_level == 2 + @test schema.root.path == String[] && schema.root.column_index == 0 +end + +@testset "schema validation" begin + required = MD.FieldRepetitionType.REQUIRED + @test_throws Parquet.FormatError Parquet.Schema(MD.SchemaElement[]) + @test_throws Parquet.FormatError Parquet.Schema([schemaelement("root"; type=MD.Type.INT32)]) + @test_throws Parquet.FormatError Parquet.Schema([schemaelement("root")]) + @test Parquet.Schema([ + schemaelement("root"; repetition=MD.FieldRepetitionType.REQUIRED, + children=Int32(0)), + ]).root.element.repetition_type == MD.FieldRepetitionType.REQUIRED + for repetition in (MD.FieldRepetitionType.OPTIONAL, MD.FieldRepetitionType.REPEATED, + MD.FieldRepetitionType.T(Int32(99))) + @test_throws Parquet.FormatError Parquet.Schema([ + schemaelement("root"; repetition=repetition, children=Int32(0)), + ]) + end + @test_throws Parquet.FormatError Parquet.Schema([schemaelement("root"; children=Int32(-1))]) + @test_throws Parquet.FormatError Parquet.Schema([ + schemaelement("root"; children=Int32(1)), + schemaelement("leaf"; type=MD.Type.INT32), + ]) + @test_throws Parquet.FormatError Parquet.Schema([ + schemaelement("root"; children=Int32(1)), + schemaelement("leaf"; type=MD.Type.INT32, repetition=required, children=Int32(1)), + ]) + @test_throws Parquet.FormatError Parquet.Schema([ + schemaelement("root"; children=Int32(1)), + schemaelement("fixed"; type=MD.Type.FIXED_LEN_BYTE_ARRAY, repetition=required), + ]) + oversized = [ + schemaelement("root"; children=Int32(1)), + schemaelement("fixed"; type=MD.Type.FIXED_LEN_BYTE_ARRAY, + repetition=required, length=Int32(1024)), + ] + error = try + Parquet.Schema(oversized; limits=Parquet.Limits(max_string_bytes=16)) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :string_bytes + @test error.requested == 1024 + @test error.maximum == 16 + @test_throws Parquet.FormatError Parquet.Schema([ + schemaelement("root"; children=Int32(2)), + schemaelement("leaf"; type=MD.Type.INT32, repetition=required), + ]) + malformedbudget = Parquet._LiveByteBudget(Parquet.Limits()) + @test_throws Parquet.FormatError Parquet.Schema([ + schemaelement("root"; children=Int32(1_000_000)), + ]; budget=malformedbudget) + @test Parquet._budgetused(malformedbudget) < 1024 + @test_throws Parquet.FormatError Parquet.Schema([ + schemaelement("root"; children=Int32(0)), + schemaelement("extra"; type=MD.Type.INT32, repetition=required), + ]) + @test_throws Parquet.LimitError Parquet.Schema([ + schemaelement("root"; children=Int32(1)), + schemaelement("group"; repetition=required, children=Int32(1)), + schemaelement("leaf"; type=MD.Type.INT32, repetition=required), + ]; limits=Parquet.Limits(max_metadata_depth=2)) +end + +@testset "iterative schema limits and rollback" begin + required = MD.FieldRepetitionType.REQUIRED + chain = MD.SchemaElement[ + schemaelement("root"; children=Int32(1)), + schemaelement("group"; repetition=required, children=Int32(1)), + schemaelement("leaf"; type=MD.Type.INT32, repetition=required), + ] + exactdepth = Parquet.Schema(chain; + limits=Parquet.Limits(max_metadata_depth=3)) + @test exactdepth.leaves[1].path == ["group", "leaf"] + deptherror = try + Parquet.Schema(chain; limits=Parquet.Limits(max_metadata_depth=2)) + nothing + catch err + err + end + @test deptherror isa Parquet.LimitError + @test deptherror.resource == :metadata_depth + @test deptherror.requested == 3 + @test deptherror.maximum == 2 + + @test length(Parquet.Schema(chain; + limits=Parquet.Limits(max_container_elements=3)).leaves) == 1 + nodeerror = try + Parquet.Schema(chain; + limits=Parquet.Limits(max_container_elements=2)) + nothing + catch err + err + end + @test nodeerror isa Parquet.LimitError + @test nodeerror.resource == :container_elements + @test nodeerror.requested == 3 + @test nodeerror.maximum == 2 + + retainedlimits = Parquet.Limits() + retainedbudget = Parquet._LiveByteBudget(retainedlimits) + Parquet._reserve!(retainedbudget, 64) + retained = Parquet.Schema(MD.SchemaElement[ + schemaelement("root"; children=Int32(1)), + schemaelement("leaf"; type=MD.Type.INT32, repetition=required), + ]; limits=retainedlimits, budget=retainedbudget) + expectedretained = + Parquet._materializedarraybytes(Parquet.SchemaNode, 2) + + Parquet._materializedarraybytes(String, 0) + + Parquet._materializedarraybytes(Parquet.SchemaNode, 1) + + Parquet._materializedarraybytes(String, 1) + + Parquet._materializedarraybytes(Parquet.SchemaNode, 0) + + 3 * Parquet._MATERIALIZED_OBJECT_BYTES + @test retained.leaves[1].column_index == 1 + @test Parquet._budgetused(retainedbudget) == 64 + expectedretained + + unclaimed = MD.SchemaElement[ + schemaelement("root"; children=Int32(0)), + schemaelement("extra"; type=MD.Type.INT32, repetition=required), + ] + unclaimederror = try + Parquet.Schema(unclaimed; + limits=Parquet.Limits(max_container_elements=1)) + nothing + catch err + err + end + @test unclaimederror isa Parquet.FormatError + @test occursin("unclaimed elements", unclaimederror.message) + + malformed = MD.SchemaElement[ + schemaelement("root"; children=Int32(1)), + schemaelement("bad"; repetition=required, children=Int32(1)), + ] + limits = Parquet.Limits(max_metadata_depth=1) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + malformederror = try + Parquet.Schema(malformed; limits=limits, budget=budget) + nothing + catch err + err + end + @test malformederror isa Parquet.FormatError + @test occursin("declares more direct children", malformederror.message) + @test Parquet._budgetused(budget) == 64 + + limitbudget = Parquet._LiveByteBudget(Parquet.Limits( + max_metadata_depth=2)) + Parquet._reserve!(limitbudget, 64) + @test_throws Parquet.LimitError Parquet.Schema(chain; + limits=Parquet.Limits(max_metadata_depth=2), budget=limitbudget) + @test Parquet._budgetused(limitbudget) == 64 +end + +@testset "official corpus schemas" begin + corpus = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + datadir = joinpath(corpus, "data") + if !isdir(datadir) + @warn "parquet-testing corpus not found; skipping schema corpus tests" corpus + else + count = 0 + for name in sort(readdir(datadir)) + endswith(name, ".parquet") || continue + file = Parquet.File(joinpath(datadir, name)) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + close(file) + schema = Parquet.Schema(metadata) + @test !isempty(schema.leaves) + @test all(length(group.columns) == length(schema.leaves) for group in metadata.row_groups) + count += 1 + end + @test count >= 65 + end +end diff --git a/test/source.jl b/test/source.jl new file mode 100644 index 0000000..f493a93 --- /dev/null +++ b/test/source.jl @@ -0,0 +1,266 @@ +struct ShiftedExactBytes <: AbstractVector{UInt8} + bytes::Vector{UInt8} +end + +struct UnitRangeExactBytes <: AbstractVector{UInt8} + bytes::Vector{UInt8} +end + +function Base.IndexStyle(::Type{ShiftedExactBytes}) + return IndexLinear() +end + +function Base.size(bytes::ShiftedExactBytes) + return (length(bytes.bytes),) +end + +function Base.axes(bytes::ShiftedExactBytes) + return (2:(length(bytes.bytes) + 1),) +end + +function Base.getindex(bytes::ShiftedExactBytes, index::Int) + checkbounds(bytes, index) + return bytes.bytes[index - 1] +end + +function Base.IndexStyle(::Type{UnitRangeExactBytes}) + return IndexLinear() +end + +function Base.size(bytes::UnitRangeExactBytes) + return (length(bytes.bytes),) +end + +function Base.axes(bytes::UnitRangeExactBytes) + return (1:length(bytes.bytes),) +end + +function Base.getindex(bytes::UnitRangeExactBytes, index::Int) + checkbounds(bytes, index) + return bytes.bytes[index] +end + +mutable struct SourceCallbackSentinel <: Exception + id::Int +end + +mutable struct ExactReadSource <: Parquet.AbstractSource + lengthvalue::Any + readvalue::Any + lengthcalls::Int + readcalls::Int +end + +function Parquet.sourcelength(source::ExactReadSource) + source.lengthcalls += 1 + source.lengthvalue isa Exception && throw(source.lengthvalue) + return source.lengthvalue +end + +function Parquet.readrange(source::ExactReadSource, offset::Integer, + count::Integer) + source.readcalls += 1 + source.readvalue isa Exception && throw(source.readvalue) + return source.readvalue +end + +mutable struct FailingPathIO <: IO + closed::Bool + failure::SourceCallbackSentinel +end + +function Base.filesize(io::FailingPathIO) + throw(io.failure) +end + +function Base.close(io::FailingPathIO) + io.closed = true + return +end + +function Base.isopen(io::FailingPathIO) + return !io.closed +end + +struct FailingSourcePath <: AbstractString + io::FailingPathIO +end + +function Base.open(path::FailingSourcePath, mode::AbstractString) + return path.io +end + +@testset "byte sources" begin + bytes = UInt8[0x10, 0x20, 0x30, 0x40] + src = Parquet.source(bytes) + @test Parquet.sourcelength(src) == 4 + @test Parquet.concurrentreads(src) + slice = Parquet.readrange(src, 1, 2) + @test collect(slice) == UInt8[0x20, 0x30] + @test_throws BoundsError Parquet.readrange(src, 3, 2) + @test_throws BoundsError Parquet.readrange(src, typemax(Int64), 1) + @test_throws ArgumentError Parquet.readrange(src, 0, -1) + Parquet.close!(src) + Parquet.close!(src) + @test_throws ArgumentError slice[1] + @test_throws ArgumentError Parquet.sourcelength(src) + + shifted = Parquet.source(ShiftedExactBytes(bytes)) + shiftedslice = Parquet.readrange(shifted, 1, 2) + @test axes(shiftedslice) == (Base.OneTo(2),) + @test collect(shiftedslice) == UInt8[0x20, 0x30] + @test Parquet._contiguous(shiftedslice) == UInt8[0x20, 0x30] + Parquet.close!(shifted) +end + +@testset "exact source adapter contract" begin + valid = ExactReadSource(Int64(4), UInt8[0x01, 0x02], 0, 0) + @test Parquet._checkedsourcelength(valid) == 4 + @test Parquet._readrangeexact(valid, Int64(4), Int64(1), Int64(2)) == + UInt8[0x01, 0x02] + @test valid.lengthcalls == 1 + @test valid.readcalls == 1 + + for value in (UInt8[0x01], UInt8[0x01, 0x02, 0x03], + Int8[0x01, 0x02], ShiftedExactBytes(UInt8[0x01, 0x02]), + UnitRangeExactBytes(UInt8[0x01, 0x02])) + source = ExactReadSource(Int64(4), value, 0, 0) + @test_throws ArgumentError Parquet._readrangeexact(source, Int64(4), + Int64(1), Int64(2)) + @test source.readcalls == 1 + end + + sentinel = SourceCallbackSentinel(1) + throwing = ExactReadSource(Int64(4), sentinel, 0, 0) + error = try + Parquet._readrangeexact(throwing, Int64(4), Int64(1), Int64(2)) + nothing + catch err + err + end + @test error === sentinel + @test throwing.readcalls == 1 + + untouched = ExactReadSource(Int64(4), UInt8[], 0, 0) + @test_throws ArgumentError Parquet._readrangeexact(untouched, Int64(4), + Int64(-1), Int64(1)) + @test_throws ArgumentError Parquet._readrangeexact(untouched, Int64(4), + Int64(0), Int64(-1)) + @test_throws ArgumentError Parquet._readrangeexact(untouched, Int64(4), + typemax(Int64), Int64(1)) + @test_throws ArgumentError Parquet._readrangeexact(untouched, Int64(4), + Int64(3), Int64(2)) + @test untouched.readcalls == 0 + + unsigned = ExactReadSource(UInt64(4), UInt8[], 0, 0) + @test Parquet._checkedsourcelength(unsigned) == 4 + wide = ExactReadSource(UInt128(4), UInt8[], 0, 0) + @test Parquet._checkedsourcelength(wide) == 4 + wrongtype = ExactReadSource(4.0, UInt8[], 0, 0) + @test_throws ArgumentError Parquet._checkedsourcelength(wrongtype) + boolean = ExactReadSource(true, UInt8[], 0, 0) + @test_throws ArgumentError Parquet._checkedsourcelength(boolean) + oversized = ExactReadSource(UInt128(typemax(Int64)) + UInt128(1), + UInt8[], 0, 0) + @test_throws ArgumentError Parquet._checkedsourcelength(oversized) + negative = ExactReadSource(Int64(-1), UInt8[], 0, 0) + @test_throws ArgumentError Parquet._checkedsourcelength(negative) + lengthsentinel = SourceCallbackSentinel(2) + lengththrowing = ExactReadSource(lengthsentinel, UInt8[], 0, 0) + lengtherror = try + Parquet._checkedsourcelength(lengththrowing) + nothing + catch err + err + end + @test lengtherror === lengthsentinel +end + + +@testset "budgeted IO buffering" begin + limits = Parquet.Limits(max_materialized_bytes=1000) + budget = Parquet._LiveByteBudget(limits) + oversized = IOBuffer(zeros(UInt8, 2000)) + @test_throws Parquet.LimitError Parquet.source(oversized; budget=budget) + @test Parquet._budgetused(budget) == 0 + @test isopen(oversized) + @test_throws Parquet.LimitError Parquet.Table( + IOBuffer(zeros(UInt8, 2000)); limits=limits) + + smallbudget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(smallbudget, Int64(17)) + src = Parquet.source(IOBuffer(zeros(UInt8, 12)); budget=smallbudget) + @test Parquet.sourcelength(src) == 12 + @test Parquet._budgetused(smallbudget) == + 17 + Parquet._materializedarraybytes(UInt8, 12) + close(src) + @test Parquet._budgetused(smallbudget) == 17 + close(src) + @test Parquet._budgetused(smallbudget) == 17 +end + +@testset "mapped file source" begin + path, io = mktemp() + try + write(io, UInt8[0x50, 0x41, 0x52, 0x31]) + close(io) + src = Parquet.source(path) + @test collect(Parquet.readrange(src, 0, 4)) == UInt8[0x50, 0x41, 0x52, 0x31] + Parquet.close!(src) + @test_throws ArgumentError Parquet.readrange(src, 0, 1) + finally + isopen(io) && close(io) + # `close!` releases the descriptor, but the mapping itself lives until the + # garbage collector finalizes it, so Windows can still hold the file open + # here. That is the documented source lifetime, not a leak, so treat a + # refused removal of this temporary file as acceptable. + GC.gc() + try + rm(path; force=true) + catch err + err isa Base.IOError || rethrow() + end + end +end + +@testset "path setup cleanup" begin + sentinel = SourceCallbackSentinel(3) + io = FailingPathIO(false, sentinel) + error = try + Parquet.source(FailingSourcePath(io)) + nothing + catch err + err + end + @test error === sentinel + @test !isopen(io) +end + +@testset "close guards are atomic" begin + # Regression: closed/materializedcharge were plain fields, so concurrent + # close! calls could both pass the guard and double-release the charge. + src = Parquet.source(UInt8[0x00]) + region = src.region + @test Base.isfieldatomic(typeof(region), :closed) + @test Base.isfieldatomic(typeof(region), :materializedcharge) + Parquet.close!(src) + Parquet.close!(src) + + limits = Parquet.Limits(max_materialized_bytes=1024) + budget = Parquet._LiveByteBudget(limits) + owned = Parquet.source(IOBuffer(zeros(UInt8, 12)); budget=budget) + @test Parquet._budgetused(budget) > 0 + @sync for _ in 1:32 + errormonitor(Threads.@spawn Parquet.close!(owned)) + end + @test Parquet._budgetused(budget) == 0 + + file = Parquet.File(Parquet._encodefile((a=Int32[1],))) + @test Base.isfieldatomic(typeof(file), :closed) + close(file) + close(file) + table = Parquet.Table(Parquet._encodefile((a=Int32[1],))) + @test Base.isfieldatomic(typeof(table), :closed) + close(table) + close(table) +end diff --git a/test/statistics.jl b/test/statistics.jl new file mode 100644 index 0000000..8bd3c53 --- /dev/null +++ b/test/statistics.jl @@ -0,0 +1,845 @@ +using SHA + +const STMD = Parquet.Metadata +const STTH = Parquet.Thrift + +struct STBlockingOrders <: AbstractVector{STMD.ColumnOrder} + values::Vector{STMD.ColumnOrder} + entered::Channel{Nothing} + release::Channel{Nothing} +end + +function Base.size(orders::STBlockingOrders) + return size(orders.values) +end + +function Base.getindex(orders::STBlockingOrders, index::Int) + put!(orders.entered, nothing) + take!(orders.release) + return orders.values[index] +end + +function st_leaf(type; name="value", width=nothing, logical=nothing, + converted=nothing, precision=nothing, scale=nothing) + return STMD.SchemaElement(type_=type, type_length=width, + repetition_type=STMD.FieldRepetitionType.OPTIONAL, name=name, + logicalType=logical, converted_type=converted, precision=precision, + scale=scale) +end + +function st_schema(leaves::Vector{STMD.SchemaElement}) + root = STMD.SchemaElement(repetition_type=STMD.FieldRepetitionType.REQUIRED, + name="schema", num_children=Int32(length(leaves))) + return Parquet.Schema(vcat(STMD.SchemaElement[root], leaves)) +end + +function st_metadata(leaf::STMD.SchemaElement, statistics=nothing; + total::Int64=4, path=String[leaf.name]) + return STMD.ColumnMetaData(type_=leaf.type_, encodings=[STMD.Encoding.PLAIN], + path_in_schema=path, codec=STMD.CompressionCodec.UNCOMPRESSED, + num_values=total, total_uncompressed_size=Int64(0), + total_compressed_size=Int64(0), data_page_offset=Int64(0), + statistics=statistics) +end + +function st_type_order() + return STMD.ColumnOrder(TYPE_ORDER=STMD.TypeDefinedOrder()) +end + +function st_ieee_order() + return STMD.ColumnOrder(IEEE_754_TOTAL_ORDER=STMD.IEEE754TotalOrder()) +end + +function st_unknown_order() + raw = STTH.RawField(77, STTH.STRUCT, UInt8[0x00]) + return STMD.ColumnOrder(unknown_fields=(raw,)) +end + +function st_le16(bits::UInt16) + return UInt8[UInt8(bits & 0xff), UInt8(bits >> 8)] +end + +function st_le32(bits::UInt32) + return UInt8[UInt8(bits & 0xff), UInt8((bits >> 8) & 0xff), + UInt8((bits >> 16) & 0xff), UInt8(bits >> 24)] +end + +function st_le64(bits::UInt64) + return UInt8[UInt8(bits & 0xff), UInt8((bits >> 8) & 0xff), + UInt8((bits >> 16) & 0xff), UInt8((bits >> 24) & 0xff), + UInt8((bits >> 32) & 0xff), UInt8((bits >> 40) & 0xff), + UInt8((bits >> 48) & 0xff), UInt8(bits >> 56)] +end + +function st_i32(value::Integer) + return st_le32(reinterpret(UInt32, Int32(value))) +end + +function st_i64(value::Integer) + return st_le64(reinterpret(UInt64, Int64(value))) +end + +function st_f32(value::Float32) + return st_le32(reinterpret(UInt32, value)) +end + +function st_f64(value::Float64) + return st_le64(reinterpret(UInt64, value)) +end + +function st_f16bits(bits::Integer) + return st_le16(UInt16(bits)) +end + +function st_facts(leaf::STMD.SchemaElement, statistics=nothing; + order=st_type_order(), created_by="Parquet.jl version 1.0.0", + total::Int64=4, limits=Parquet.Limits(), budget=nothing) + schema = st_schema(STMD.SchemaElement[leaf]) + orders = order === nothing ? nothing : STMD.ColumnOrder[order] + metadata = st_metadata(leaf, statistics; total=total) + budget === nothing && return Parquet._statisticsfacts(schema, 1, created_by, + orders, metadata; limits=limits) + return Parquet._statisticsfacts(schema, 1, created_by, orders, metadata; + limits=limits, budget=budget) +end + +function st_modern(lower, upper; nulls=nothing, nans=nothing, distinct=nothing, + lower_exact=nothing, upper_exact=nothing) + return STMD.Statistics(min_value=lower, max_value=upper, + null_count=nulls, nan_count=nans, distinct_count=distinct, + is_min_value_exact=lower_exact, is_max_value_exact=upper_exact) +end + +function st_deprecated(lower, upper; nulls=nothing, nans=nothing, + distinct=nothing) + return STMD.Statistics(min=lower, max=upper, null_count=nulls, + nan_count=nans, distinct_count=distinct) +end + +function st_decimaltype(precision::Integer) + return STMD.LogicalType(DECIMAL=STMD.DecimalType(scale=Int32(0), + precision=Int32(precision))) +end + +@testset "reader statistics physical and logical orders" begin + physical = ( + (st_leaf(STMD.Type.BOOLEAN), :boolean), + (st_leaf(STMD.Type.INT32), :signed), + (st_leaf(STMD.Type.INT64), :signed), + (st_leaf(STMD.Type.INT96), :undefined), + (st_leaf(STMD.Type.FLOAT), :floating), + (st_leaf(STMD.Type.DOUBLE), :floating), + (st_leaf(STMD.Type.BYTE_ARRAY), :unsigned_bytes), + (st_leaf(STMD.Type.FIXED_LEN_BYTE_ARRAY; width=Int32(3)), :unsigned_bytes), + ) + for (leaf, comparison) in physical + facts = st_facts(leaf) + @test facts.order.comparison == comparison + end + + signed = STMD.LogicalType(INTEGER=STMD.IntType(bitWidth=Int8(8), + isSigned=true)) + unsigned = STMD.LogicalType(INTEGER=STMD.IntType(bitWidth=Int8(32), + isSigned=false)) + stringtype = STMD.LogicalType(STRING=STMD.StringType()) + enumtype = STMD.LogicalType(ENUM=STMD.EnumType()) + uuidtype = STMD.LogicalType(UUID=STMD.UUIDType()) + date = STMD.LogicalType(DATE=STMD.DateType()) + millis = STMD.TimeUnit(MILLIS=STMD.MilliSeconds()) + time = STMD.LogicalType(TIME=STMD.TimeType(isAdjustedToUTC=false, unit=millis)) + timestamp = STMD.LogicalType(TIMESTAMP=STMD.TimestampType( + isAdjustedToUTC=false, unit=millis)) + float16 = STMD.LogicalType(FLOAT16=STMD.Float16Type()) + logical = ( + (st_leaf(STMD.Type.INT32; logical=signed), :signed), + (st_leaf(STMD.Type.INT32; logical=unsigned), :unsigned), + (st_leaf(STMD.Type.BYTE_ARRAY; logical=stringtype), :unsigned_bytes), + (st_leaf(STMD.Type.BYTE_ARRAY; logical=enumtype), :unsigned_bytes), + (st_leaf(STMD.Type.FIXED_LEN_BYTE_ARRAY; width=Int32(16), logical=uuidtype), + :unsigned_bytes), + (st_leaf(STMD.Type.INT32; logical=date), :signed), + (st_leaf(STMD.Type.INT32; logical=time), :signed), + (st_leaf(STMD.Type.INT64; logical=timestamp), :signed), + (st_leaf(STMD.Type.FIXED_LEN_BYTE_ARRAY; width=Int32(2), logical=float16), + :floating), + ) + for (leaf, comparison) in logical + @test st_facts(leaf).order.comparison == comparison + end + + undefined = ( + st_leaf(STMD.Type.FIXED_LEN_BYTE_ARRAY; width=Int32(12), + converted=STMD.ConvertedType.INTERVAL), + st_leaf(STMD.Type.INT32; logical=STMD.LogicalType(UNKNOWN=STMD.NullType())), + st_leaf(STMD.Type.BYTE_ARRAY; + logical=STMD.LogicalType(VARIANT=STMD.VariantType())), + st_leaf(STMD.Type.BYTE_ARRAY; + logical=STMD.LogicalType(GEOMETRY=STMD.GeometryType())), + st_leaf(STMD.Type.BYTE_ARRAY; + logical=STMD.LogicalType(GEOGRAPHY=STMD.GeographyType())), + ) + for leaf in undefined + @test st_facts(leaf).order.comparison == :undefined + ignored = st_facts(leaf, st_modern(fill(UInt8(0), + something(Parquet._statisticplainwidth(leaf), 1)), fill(UInt8(0), + something(Parquet._statisticplainwidth(leaf), 1)))) + @test ignored.lower.state == :unknown + @test ignored.lower.reason == :undefined_order + end + + modernwins = st_leaf(STMD.Type.BYTE_ARRAY; logical=stringtype, + converted=STMD.ConvertedType.INTERVAL) + @test st_facts(modernwins).order.comparison == :unsigned_bytes + unsupported = st_leaf(STMD.Type.BYTE_ARRAY; + logical=STMD.LogicalType(VARIANT=STMD.VariantType())) + @test st_facts(unsupported, + st_modern(UInt8[0x01], UInt8[0x02])).lower.state == :unknown + + futurelogical = STMD.LogicalType(unknown_fields=( + STTH.RawField(91, STTH.STRUCT, UInt8[0x00]),)) + futurefloat = st_leaf(STMD.Type.FLOAT; logical=futurelogical) + futurestats = st_modern(st_f32(-1.0f0), st_f32(1.0f0); + nulls=Int64(0), nans=Int64(0)) + futuretype = st_facts(futurefloat, futurestats) + @test futuretype.order.comparison == :undefined + @test futuretype.lower.reason == :undefined_order + @test futuretype.nan_count.value == 0 + futureieee = st_facts(futurefloat, futurestats; order=st_ieee_order()) + @test futureieee.order.comparison == :ieee_total_order + @test futureieee.lower.state == :known + @test futureieee.upper.state == :known +end + +@testset "reader statistics families, identity, and exactness" begin + leaf = st_leaf(STMD.Type.INT32) + lower = st_i32(-3) + upper = st_i32(8) + deprecated_lower = st_i32(-100) + statistics = STMD.Statistics(min=deprecated_lower, max=st_i32(100), + min_value=lower, is_min_value_exact=true) + facts = st_facts(leaf, statistics) + @test facts.family == :modern + @test facts.lower.state == :known + @test facts.lower.raw === lower + @test facts.lower.exactness == :exact + @test facts.upper.state == :absent + @test facts.upper.raw === nothing + + facts = st_facts(leaf, st_modern(lower, upper; lower_exact=false, + upper_exact=nothing)) + @test facts.lower.exactness == :inexact + @test facts.upper.exactness == :unknown + + facts = st_facts(leaf, st_deprecated(lower, upper)) + @test facts.family == :deprecated + @test facts.lower.state == :known + @test facts.lower.exactness == :unknown + @test st_facts(leaf, st_deprecated(lower, upper); order=nothing).lower.state == + :known + + unsigned = STMD.LogicalType(INTEGER=STMD.IntType(bitWidth=Int8(32), + isSigned=false)) + unsignedleaf = st_leaf(STMD.Type.INT32; logical=unsigned) + facts = st_facts(unsignedleaf, st_deprecated(st_le32(0x00000001), + st_le32(0xffffffff))) + @test facts.lower.state == :unknown + @test facts.lower.reason == :deprecated_order_mismatch + + contradictory = st_facts(leaf, st_modern(st_i32(9), st_i32(2))) + @test contradictory.lower.state == :unknown + @test contradictory.upper.state == :unknown + @test contradictory.lower.reason == :contradictory_bounds + flagswithoutbounds = st_facts(leaf, + STMD.Statistics(is_min_value_exact=true, is_max_value_exact=false)) + @test flagswithoutbounds.lower.state == :absent + @test flagswithoutbounds.upper.state == :absent +end + +@testset "reader statistics count validation" begin + leaf = st_leaf(STMD.Type.FLOAT) + facts = st_facts(leaf, st_modern(nothing, nothing; nulls=Int64(0), + nans=Int64(0), distinct=Int64(0))) + @test facts.null_count == Parquet._StatisticCountFact(:known, 0) + @test facts.nan_count == Parquet._StatisticCountFact(:known, 0) + @test facts.distinct_count == Parquet._StatisticCountFact(:known, 0) + + facts = st_facts(leaf, STMD.Statistics()) + @test facts.null_count.state == :absent + @test facts.nan_count.state == :absent + @test facts.distinct_count.state == :absent + + for statistics in ( + STMD.Statistics(null_count=Int64(-1)), + STMD.Statistics(null_count=Int64(5)), + STMD.Statistics(nan_count=Int64(-1)), + STMD.Statistics(nan_count=Int64(5)), + STMD.Statistics(distinct_count=Int64(-1)), + STMD.Statistics(distinct_count=Int64(5)), + STMD.Statistics(null_count=Int64(3), nan_count=Int64(2)), + STMD.Statistics(null_count=Int64(2), distinct_count=Int64(3)), + ) + @test_throws Parquet.FormatError st_facts(leaf, statistics) + end + @test_throws Parquet.FormatError st_facts(st_leaf(STMD.Type.INT32), + STMD.Statistics(nan_count=Int64(0))) + @test_throws Parquet.FormatError st_facts(leaf, + STMD.Statistics(null_count=typemax(Int64), nan_count=typemax(Int64)); + total=typemax(Int64)) + + allnull = st_facts(st_leaf(STMD.Type.INT32), + st_modern(st_i32(1), st_i32(2); nulls=Int64(4))) + @test allnull.lower.state == :unknown + @test allnull.lower.reason == :no_non_null + + nested = st_facts(st_leaf(STMD.Type.INT32), + STMD.Statistics(null_count=Int64(3)); total=Int64(7)) + @test nested.null_count.value == 3 + empty = st_facts(st_leaf(STMD.Type.INT32), + st_modern(st_i32(1), st_i32(2)); total=Int64(0)) + @test empty.lower.reason == :no_non_null +end + +@testset "reader statistics column orders and entry precedence" begin + leaves = STMD.SchemaElement[st_leaf(STMD.Type.INT32; name="a"), + st_leaf(STMD.Type.FLOAT; name="b")] + schema = st_schema(leaves) + metadata = st_metadata(leaves[1], nothing; path=["a"]) + @test_throws Parquet.ArgumentError Parquet._statisticsfacts(schema, 0, + nothing, nothing, metadata) + @test_throws Parquet.ArgumentError Parquet._statisticsfacts(schema, 3, + nothing, nothing, metadata) + @test_throws Parquet.FormatError Parquet._statisticsfacts(schema, 1, + nothing, STMD.ColumnOrder[st_type_order()], metadata) + @test_throws Parquet.FormatError Parquet._statisticsfacts(schema, 1, + nothing, STMD.ColumnOrder[st_type_order(), st_type_order(), st_type_order()], + metadata) + @test_throws ArgumentError Parquet._statisticsfacts(schema, 1, + nothing, STMD.ColumnOrder[st_type_order(), st_ieee_order()], metadata; + limits=Parquet.Limits(max_statistics_value_bytes=-1)) + @test_throws Parquet.FormatError Parquet._statisticsfacts(schema, 1, + nothing, nothing, st_metadata(leaves[2], nothing; path=["a"])) + @test_throws Parquet.FormatError Parquet._statisticsfacts(schema, 1, + nothing, nothing, st_metadata(leaves[1], nothing; path=["wrong"])) + @test_throws Parquet.FormatError Parquet._statisticsfacts(schema, 1, + nothing, nothing, st_metadata(leaves[1], nothing; total=Int64(-1), path=["a"])) + + entered = Channel{Nothing}(1) + release = Channel{Nothing}(1) + blocking = STBlockingOrders(STMD.ColumnOrder[st_type_order()], entered, release) + single = st_schema(STMD.SchemaElement[leaves[1]]) + budget = Parquet._LiveByteBudget( + Parquet.Limits(max_materialized_bytes=Int64(1024))) + task = errormonitor(Threads.@spawn begin + try + return Parquet._statisticsfacts(single, 1, nothing, blocking, + st_metadata(leaves[1], nothing; path=["wrong"]); budget=budget) + catch err + return err + end + end) + take!(entered) + Parquet._reserve!(budget, 100) + put!(release, nothing) + @test fetch(task) isa Parquet.FormatError + @test Parquet._budgetused(budget) == 100 + Parquet._release!(budget, 100) + + badorders = STMD.ColumnOrder[st_type_order(), st_ieee_order()] + @test Parquet._statisticsfacts(schema, 1, nothing, badorders, metadata).order.declared == + :type_order + illegal = STMD.ColumnOrder[st_ieee_order(), st_ieee_order()] + @test_throws Parquet.FormatError Parquet._statisticsfacts(schema, 2, + nothing, illegal, st_metadata(leaves[2], nothing; path=["b"])) + + empty = STMD.ColumnOrder() + @test_throws Parquet.FormatError st_facts(leaves[1], nothing; order=empty) + unknown = st_facts(leaves[1], st_modern(st_i32(1), st_i32(2)); + order=st_unknown_order()) + @test unknown.order.declared == :unknown + @test unknown.lower.state == :unknown + @test unknown.null_count.state == :absent + + missing = st_facts(leaves[1], st_modern(st_i32(1), st_i32(2)); order=nothing) + @test missing.order.declared == :absent + @test missing.lower.reason == :missing_order + + ieeeinteger = st_facts(leaves[2], nothing; order=st_ieee_order()) + @test ieeeinteger.order.comparison == :ieee_total_order +end + +@testset "reader statistics widths, limits, and semantic validation" begin + fixed = st_leaf(STMD.Type.FIXED_LEN_BYTE_ARRAY; width=Int32(4)) + wrong = st_modern(UInt8[0x01, 0x02, 0x03], UInt8[0x01, 0x02, 0x03, 0x04]) + @test_throws Parquet.FormatError st_facts(fixed, wrong; + limits=Parquet.Limits(max_statistics_value_bytes=Int64(0))) + + variable = st_leaf(STMD.Type.BYTE_ARRAY; + logical=STMD.LogicalType(STRING=STMD.StringType())) + invalidutf8 = UInt8[0xff] + exact = st_facts(variable, st_modern(invalidutf8, invalidutf8); + limits=Parquet.Limits(max_statistics_value_bytes=Int64(1))) + @test exact.lower.state == :unknown + @test exact.lower.reason == :invalid_value + over = st_facts(variable, st_modern(invalidutf8, invalidutf8); + limits=Parquet.Limits(max_statistics_value_bytes=Int64(0))) + @test over.lower.state == :unknown + @test over.lower.reason == :over_limit + @test over.lower.raw === invalidutf8 + overmissing = st_facts(variable, st_modern(invalidutf8, invalidutf8); + order=nothing, limits=Parquet.Limits(max_statistics_value_bytes=Int64(0))) + @test overmissing.lower.reason == :over_limit + @test_throws ArgumentError st_facts(variable, nothing; + limits=Parquet.Limits(max_statistics_value_bytes=Int64(-1))) + + rawleaf = st_leaf(STMD.Type.BYTE_ARRAY) + raw = UInt8[0x00, 0xff, 0x80] + rawfacts = st_facts(rawleaf, st_modern(raw, raw)) + @test rawfacts.lower.state == :known + @test rawfacts.lower.raw === raw + emptyraw = UInt8[] + @test st_facts(rawleaf, st_modern(emptyraw, emptyraw); + limits=Parquet.Limits(max_statistics_value_bytes=Int64(0))).lower.state == + :known + exactraw = UInt8[0x80] + @test st_facts(rawleaf, st_modern(exactraw, exactraw); + limits=Parquet.Limits(max_statistics_value_bytes=Int64(1))).lower.state == + :known + + boolean = st_leaf(STMD.Type.BOOLEAN) + @test st_facts(boolean, st_modern(UInt8[0x00], UInt8[0x01])).lower.state == :known + @test st_facts(boolean, st_modern(UInt8[0x02], UInt8[0x02])).lower.state == :unknown + @test_throws Parquet.FormatError st_facts(boolean, + st_modern(UInt8[], UInt8[0x01])) + + signed8 = st_leaf(STMD.Type.INT32; logical=STMD.LogicalType( + INTEGER=STMD.IntType(bitWidth=Int8(8), isSigned=true))) + unsigned8 = st_leaf(STMD.Type.INT32; logical=STMD.LogicalType( + INTEGER=STMD.IntType(bitWidth=Int8(8), isSigned=false))) + @test st_facts(signed8, st_modern(st_i32(-128), st_i32(127))).lower.state == :known + @test st_facts(signed8, st_modern(st_i32(-129), st_i32(127))).lower.state == :unknown + @test st_facts(unsigned8, st_modern(st_le32(0x00000000), + st_le32(0x000000ff))).upper.state == :known + @test st_facts(unsigned8, st_modern(st_le32(0x00000100), + st_le32(0x00000100))).lower.state == :unknown + + millis = STMD.TimeUnit(MILLIS=STMD.MilliSeconds()) + timeleaf = st_leaf(STMD.Type.INT32; logical=STMD.LogicalType( + TIME=STMD.TimeType(isAdjustedToUTC=false, unit=millis))) + @test st_facts(timeleaf, st_modern(st_i32(0), st_i32(86_399_999))).upper.state == + :known + @test st_facts(timeleaf, st_modern(st_i32(-1), st_i32(86_400_000))).lower.state == + :unknown + + jsonleaf = st_leaf(STMD.Type.BYTE_ARRAY; + logical=STMD.LogicalType(JSON=STMD.JsonType())) + bsonleaf = st_leaf(STMD.Type.BYTE_ARRAY; + logical=STMD.LogicalType(BSON=STMD.BsonType())) + validjsonlower = Vector{UInt8}(codeunits("{\"a\":1}")) + validjsonupper = Vector{UInt8}(codeunits("{\"a\":2}")) + validjson = st_modern(validjsonlower, validjsonupper) + @test st_facts(jsonleaf, validjson).lower.state == :known + @test st_facts(jsonleaf, st_modern(UInt8[0x7b], UInt8[0x7b])).lower.state == :unknown + tinybudget = Parquet._LiveByteBudget( + Parquet.Limits(max_materialized_bytes=Int64(127))) + @test_throws Parquet.LimitError st_facts(jsonleaf, validjson; budget=tinybudget) + @test Parquet._budgetused(tinybudget) == 0 + exactbudget = Parquet._LiveByteBudget( + Parquet.Limits(max_materialized_bytes=Int64(128))) + @test st_facts(jsonleaf, validjson; budget=exactbudget).lower.state == :known + @test Parquet._budgetused(exactbudget) == 0 + emptybson = UInt8[0x05, 0x00, 0x00, 0x00, 0x00] + @test st_facts(bsonleaf, st_modern(emptybson, emptybson)).lower.state == :known + @test st_facts(bsonleaf, st_modern(UInt8[0x00], UInt8[0x00])).lower.state == :unknown +end + +@testset "reader statistics decimal orders and precision" begin + int32leaf = st_leaf(STMD.Type.INT32; logical=st_decimaltype(3)) + int64leaf = st_leaf(STMD.Type.INT64; logical=st_decimaltype(18)) + byteleaf = st_leaf(STMD.Type.BYTE_ARRAY; logical=st_decimaltype(5)) + fixedleaf = st_leaf(STMD.Type.FIXED_LEN_BYTE_ARRAY; width=Int32(4), + logical=st_decimaltype(9)) + + @test st_facts(int32leaf, st_modern(st_i32(-999), st_i32(999))).lower.state == + :known + @test st_facts(int32leaf, st_modern(st_i32(-1000), st_i32(1000))).lower.state == + :unknown + @test st_facts(int64leaf, st_modern(st_i64(-999_999_999_999_999_999), + st_i64(999_999_999_999_999_999))).upper.state == :known + @test st_facts(int64leaf, st_modern(st_i64(typemin(Int64)), + st_i64(typemin(Int64)))).lower.state == :unknown + + negative = UInt8[0xff, 0x7f] + positive = UInt8[0x00, 0x80] + bytefacts = st_facts(byteleaf, st_modern(negative, positive)) + @test bytefacts.lower.state == :known + @test bytefacts.upper.state == :known + @test Parquet._comparestatisticvalues(byteleaf, negative, positive, :decimal) == -1 + @test Parquet._comparestatisticvalues(byteleaf, UInt8[0xff], UInt8[0xff, 0xff], + :decimal) == 0 + @test Parquet._comparestatisticvalues(byteleaf, UInt8[0xff, 0x7f], UInt8[0x80], + :decimal) == -1 + + @test st_facts(byteleaf, st_modern(UInt8[], UInt8[])).lower.state == :unknown + @test st_facts(byteleaf, st_modern(UInt8[0x0f, 0x42, 0x40], + UInt8[0x0f, 0x42, 0x40])).lower.state == :unknown + @test st_facts(fixedleaf, st_modern(UInt8[0xff, 0xff, 0xfc, 0x18], + UInt8[0x00, 0x00, 0x03, 0xe8])).lower.state == :known + + budget = Parquet._LiveByteBudget(Parquet.Limits(max_materialized_bytes=Int64(67))) + @test_throws Parquet.LimitError st_facts(byteleaf, + st_modern(UInt8[0x01, 0x02, 0x03, 0x04], UInt8[0x05]); budget=budget) + @test Parquet._budgetused(budget) == 0 + retained = Parquet._LiveByteBudget( + Parquet.Limits(max_materialized_bytes=Int64(74))) + Parquet._reserve!(retained, 7) + @test_throws Parquet.LimitError st_facts(byteleaf, + st_modern(UInt8[0x01, 0x02, 0x03, 0x04], UInt8[0x05]); budget=retained) + @test Parquet._budgetused(retained) == 7 + Parquet._release!(retained, 7) +end + +@testset "reader statistics floating policies" begin + floatleaf = st_leaf(STMD.Type.FLOAT) + doubleleaf = st_leaf(STMD.Type.DOUBLE) + float16leaf = st_leaf(STMD.Type.FIXED_LEN_BYTE_ARRAY; width=Int32(2), + logical=STMD.LogicalType(FLOAT16=STMD.Float16Type())) + + for (leaf, lower, upper) in ( + (floatleaf, st_f32(-Inf32), st_f32(Inf32)), + (doubleleaf, st_f64(-Inf), st_f64(Inf)), + (float16leaf, st_f16bits(0xfc00), st_f16bits(0x7c00)), + ) + facts = st_facts(leaf, st_modern(lower, upper); order=st_ieee_order()) + @test facts.lower.state == :known + @test facts.upper.state == :known + @test facts.order.comparison == :ieee_total_order + end + + positivezero = st_f32(0.0f0) + negativezero = st_f32(-0.0f0) + typezeros = st_facts(floatleaf, st_modern(positivezero, negativezero; + lower_exact=true, upper_exact=true)) + @test typezeros.lower.adjustment == :negative_zero + @test typezeros.upper.adjustment == :positive_zero + @test typezeros.lower.exactness == :inexact + @test typezeros.upper.exactness == :inexact + @test typezeros.lower.raw === positivezero + @test typezeros.upper.raw === negativezero + + ieeezeros = st_facts(floatleaf, st_modern(negativezero, positivezero; + nulls=Int64(0), nans=Int64(0)); order=st_ieee_order()) + @test ieeezeros.lower.state == :known + @test ieeezeros.upper.state == :known + @test Parquet._comparestatisticvalues(floatleaf, negativezero, positivezero, + :ieee_total_order) == -1 + + quietnan = st_le32(0x7fc00001) + signalingnan = st_le32(0x7f800001) + @test Parquet._comparestatisticvalues(floatleaf, quietnan, st_f32(1.0f0), + :floating) === nothing + @test Parquet._comparestatisticvalues(floatleaf, st_f32(1.0f0), quietnan, + :floating) === nothing + insufficient = st_facts(floatleaf, st_modern(quietnan, st_f32(3.0f0))) + @test insufficient.lower.state == :unknown + @test insufficient.upper.state == :known + @test insufficient.lower.reason == :nan_type_order + + allnantype = st_facts(floatleaf, st_modern(quietnan, signalingnan; + nulls=Int64(0), nans=Int64(4))) + @test allnantype.lower.reason == :all_nan_type_order + @test allnantype.upper.reason == :all_nan_type_order + + allnanieee = st_facts(floatleaf, st_modern(signalingnan, quietnan; + nulls=Int64(0), nans=Int64(4)); order=st_ieee_order()) + @test allnanieee.lower.state == :known + @test allnanieee.upper.state == :known + @test Parquet._comparestatisticvalues(floatleaf, signalingnan, quietnan, + :ieee_total_order) < 0 + + wrongallnan = st_facts(floatleaf, st_modern(st_f32(1.0f0), quietnan; + nulls=Int64(0), nans=Int64(4)); order=st_ieee_order()) + @test wrongallnan.lower.reason == :ieee_bound_kind + @test wrongallnan.upper.reason == :ieee_bound_kind + mixedcontradiction = st_facts(floatleaf, st_modern(st_f32(1.0f0), quietnan; + nulls=Int64(0), nans=Int64(1)); order=st_ieee_order()) + @test mixedcontradiction.lower.reason == :ieee_bound_kind + @test mixedcontradiction.upper.reason == :ieee_bound_kind + missingcounts = st_facts(floatleaf, st_modern(quietnan, st_f32(2.0f0)); + order=st_ieee_order()) + @test missingcounts.lower.reason == :unproven_ieee_nan + @test missingcounts.upper.state == :known + + empty = st_facts(floatleaf, st_modern(st_f32(1.0f0), st_f32(2.0f0); + nulls=Int64(4), nans=Int64(0)); order=st_ieee_order()) + @test empty.lower.reason == :no_non_null + + deprecated = st_facts(floatleaf, + st_deprecated(st_f32(-1.0f0), st_f32(1.0f0)); + created_by="parquet-cpp version 1.2.9") + @test deprecated.lower.state == :known + @test deprecated.upper.state == :known + @test deprecated.trust.state == :trusted + + seen = falses(65_536) + ordered = Vector{UInt8}(undef, 2 * 65_536) + unique = true + roundtrip = true + nancount = 0 + zerocount = 0 + for rawbits in UInt32(0):UInt32(0xffff) + bits = UInt16(rawbits) + raw = st_le16(bits) + roundtrip &= Parquet._statisticfloatbits(float16leaf, raw) == bits + key = Parquet._statisticieeekey(bits) + position = Int(key) + 1 + unique &= !seen[position] + seen[position] = true + ordered[2 * position - 1] = raw[1] + ordered[2 * position] = raw[2] + nancount += Parquet._statisticisnan(bits) + zerocount += Parquet._statisticiszero(bits) + end + @test roundtrip + @test unique + @test all(seen) + @test nancount == 2046 + @test zerocount == 2 + @test bytes2hex(SHA.sha256(ordered)) == + "61619b1a4260ee4cff6d462d21cab5a049198a27b3a9ae3e8ecfe246efc1d4b6" +end + +@testset "reader statistics producer trust" begin + decimal = STMD.LogicalType(DECIMAL=STMD.DecimalType(scale=Int32(0), + precision=Int32(3))) + signedbinary = st_leaf(STMD.Type.BYTE_ARRAY; logical=decimal) + decimalstats = st_modern(UInt8[0xff], UInt8[0x01]) + for createdby in (nothing, "", "not a parsed created-by", + "parquet-mr", "parquet-mr version 1.7.9", + "parquet-mr version 1.7", + "parquet-mr version 2147483648.0.0", + "parquet-mr version 1.8.0-rc1", + "parquet-mr version 1.8.1-2147483648", + "parquet-mr version 1.8.1-alpha.2147483648", + "parquet-mr version 1.5.0-cdh5.4.9", + "parquet-mr version 1.5.0-.", + "parquet-mr version 1.5.0-..", + "parquet-mr version 1.5.0-cdh5.5.", + "parquet-mr version 1.5.0-cdh5.5..", + "parquet-mr version 1.5.0-cdh5.2147483648.0", + "parquet-mr version 1.5.0-cdh10.0.0", + "parquet-mr version 1.8.0 (not-build metadata)", + "unrelated version 0.1.0 (not-build metadata)") + facts = st_facts(signedbinary, decimalstats; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == :parquet_251 + @test facts.null_count.state == :absent + end + for createdby in ("parquet-mr version 1.8.0", + "parquet-mr version 1.8.0 (build abc123)", + "parquet-mr version 1.8.1-2147483647", + "parquet-mr version 1.8.1+2147483648", + "parquet-mr version 1.5.0-cdh5.5.0", + "parquet-mr version 1.5.0-cdh5.5.0.", + "parquet-mr version 1.5.0-cdh5.5.0..", + "parquet-mr version 1.5.0-cdh5.5.0-SNAPSHOT", + "parquet-mr version 1.5.0-cdh5.2147483648x.0", + "parquet-mr version 1.5.0-cdh5.-1.0", + "parquet-mr version 1.5.0-cdh5. 5.0", + "parquet-mr version 1.5.0-cdh5.\u0665.0", + "parquet-mr version 1.5.0-cdh5.6.1", + "unrelated version 0.1.0") + @test st_facts(signedbinary, decimalstats; + created_by=createdby).trust.state == :trusted + end + + rawleaf = st_leaf(STMD.Type.BYTE_ARRAY) + distinct = st_modern(UInt8[0x01], UInt8[0x02]) + equal = st_modern(UInt8[0x01], UInt8[0x01]) + nobounds = st_facts(rawleaf, nothing; created_by=nothing) + @test nobounds.trust.state == :trusted + @test nobounds.trust.reason == :no_bounds + bare = st_facts(rawleaf, distinct; created_by="parquet-cpp") + @test bare.trust.state == :untrusted + @test bare.trust.reason == :parquet_251 + @test st_facts(rawleaf, equal; + created_by="parquet-cpp").trust.reason == :parquet_251 + @test st_facts(rawleaf, st_deprecated(UInt8[0x01], UInt8[0x02]); + created_by="parquet-cpp").trust.reason == :parquet_251 + for createdby in ("parquet-cpp version 1.2.9", + "parquet-cpp version 1.3.0-rc1", + "parquet-cpp version1.2.9", + "parquet-cpp\tversion\t1.2.9", + " \tparquet-cpp\nversion\n1.2.9 \r") + facts = st_facts(rawleaf, distinct; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == :parquet_cpp_pre_1_3 + end + @test st_facts(rawleaf, distinct; + created_by="parquet-cpp version 1.3.0").trust.state == :trusted + @test st_facts(rawleaf, distinct; + created_by=" \tparquet-cpp version1.3.0 \n").trust.state == :trusted + @test st_facts(rawleaf, distinct; + created_by="parquet-cpp-arrow version 1.3.0").trust.state == :trusted + @test st_facts(rawleaf, distinct; + created_by="parquet-cpp-arrow version 1.2.0").trust.state == :trusted + @test st_facts(rawleaf, distinct; + created_by="parquet-cpp version 1.3.0rc1").trust.state == :untrusted + @test st_facts(rawleaf, equal; + created_by="parquet-cpp version 1.2.9").trust.reason == + :affected_equal_bounds + oversizedequal = fill(UInt8(0x01), 2) + oversizedfacts = st_facts(rawleaf, + st_modern(oversizedequal, oversizedequal); + created_by="parquet-cpp version 1.2.9", + limits=Parquet.Limits(max_statistics_value_bytes=Int64(1))) + @test oversizedfacts.trust.reason == :parquet_cpp_pre_1_3 + @test oversizedfacts.lower.reason == :over_limit + + for createdby in ("parquet-mr version 1.9.9", + "parquet-mr version 1.10.0-rc1", + "parquet-mr version1.9.9", + "parquet-mr\tversion\t1.9.9", + " \tparquet-mr\nversion\n1.9.9 \r") + facts = st_facts(rawleaf, distinct; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == :parquet_mr_pre_1_10 + end + @test st_facts(rawleaf, distinct; + created_by="parquet-mr version 1.10.0").trust.state == :trusted + @test st_facts(rawleaf, distinct; + created_by=" \tparquet-mr\tversion\t1.10.0 \n").trust.state == :trusted + @test st_facts(rawleaf, equal; + created_by="parquet-mr version 1.9.0").trust.reason == + :affected_equal_bounds + + for createdby in ("foo\nbar version 1.0.0", + "parquet-mr\nextra version 1.7.9", + "parquet-cpp\nextra version 1.2.9") + @test !Parquet._statisticsjavaproducer(createdby).parsed + facts = st_facts(rawleaf, distinct; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == :parquet_251 + end + for terminator in ("\r", "\r\n", "\u0085", "\u2028", "\u2029") + createdby = "foo" * terminator * "bar version 1.0.0" + @test !Parquet._statisticsjavaproducer(createdby).parsed + facts = st_facts(rawleaf, distinct; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == :parquet_251 + end + for (createdby, reason) in (("parquet-mr\nversion\n1.9.9", + :parquet_mr_pre_1_10), + ("parquet-cpp\nversion\n1.2.9", :parquet_cpp_pre_1_3)) + @test Parquet._statisticsjavaproducer(createdby).parsed + facts = st_facts(rawleaf, distinct; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == reason + end + for separator in ("\r", "\r\n", "\v", "\f") + createdby = "parquet-mr" * separator * "version 1.9.9" + @test Parquet._statisticsjavaproducer(createdby).parsed + facts = st_facts(rawleaf, distinct; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == :parquet_mr_pre_1_10 + end + for separator in ("\u0085", "\u2028", "\u2029") + createdby = "parquet-mr" * separator * "version 1.8.0" + @test !Parquet._statisticsjavaproducer(createdby).parsed + facts = st_facts(rawleaf, distinct; created_by=createdby) + @test facts.trust.state == :untrusted + @test facts.trust.reason == :parquet_251 + end + for terminator in ("\n", "\r", "\r\n", "\u0085", "\u2028", "\u2029") + mrcreatedby = "parquet-mr version 1.10.0+foo" * terminator * "bar" + @test !Parquet._statisticsjavaproducer(mrcreatedby).version.present + @test st_facts(rawleaf, distinct; + created_by=mrcreatedby).trust.reason == :parquet_251 + cppcreatedby = "parquet-cpp version 1.3.0+foo" * terminator * "bar" + @test !Parquet._statisticsjavaproducer(cppcreatedby).version.present + @test st_facts(rawleaf, distinct; + created_by=cppcreatedby).trust.reason == :parquet_cpp_pre_1_3 + end + for separator in ("\v", "\f") + for createdby in ("parquet-mr version 1.10.0+foo" * separator * "bar", + "parquet-cpp version 1.3.0+foo" * separator * "bar") + @test Parquet._statisticsjavaproducer(createdby).version.present + @test st_facts(rawleaf, distinct; + created_by=createdby).trust.state == :trusted + end + end + backtracked = Parquet._statisticsjavaproducer( + "foo version (bad) bar version 1.0.0") + @test backtracked.parsed + @test backtracked.application == :other + for createdby in ("parquet-mr version (bad) x version 1.7.9", + "parquet-cpp version (bad) x version 1.2.9") + producer = Parquet._statisticsjavaproducer(createdby) + @test producer.parsed + @test producer.application == :other + @test st_facts(rawleaf, distinct; + created_by=createdby).trust.state == :trusted + end + backtrackinghostile = "parquet-mr" * repeat(" version (bad)", 8_192) * + " version 1.0.0" + @test Parquet._statisticsjavaproducer(backtrackinghostile).parsed + hostile = "parquet-cpp" * repeat(" ", 32_768) * "not-version" + @test !Parquet._statisticsjavaproducer(hostile).parsed + + signedleaf = st_leaf(STMD.Type.INT32) + @test st_facts(signedleaf, st_modern(st_i32(1), st_i32(2)); + created_by=nothing).trust.state == :trusted + unsigned = STMD.LogicalType(INTEGER=STMD.IntType(bitWidth=Int8(32), + isSigned=false)) + unsignedleaf = st_leaf(STMD.Type.INT32; logical=unsigned) + for createdby in ("parquet-cpp", "parquet-cpp version 1.2", + "parquet-cpp version 2147483648.0.0", + "parquet-cpp version 1.3.1-2147483648", "parquet-mr", + "parquet-mr version 1.9", "parquet-mr version 1.10.1-2147483648") + @test st_facts(unsignedleaf, st_modern(st_le32(0x00000001), + st_le32(0x00000002)); created_by=createdby).trust.state == :untrusted + end + for createdby in ("parquet-cpp version 1.3.1-2147483647", + "parquet-cpp version 1.3.1-2147483648x", + "parquet-cpp version 1.3.1-2147483648)", + "parquet-mr version 1.10.1-2147483647", + "parquet-mr version 1.10.1-2147483648x", + "parquet-mr version 1.10.1-2147483648)") + @test st_facts(unsignedleaf, st_modern(st_le32(0x00000001), + st_le32(0x00000002)); created_by=createdby).trust.state == :trusted + end + ieeeold = st_facts(st_leaf(STMD.Type.FLOAT), + st_modern(st_f32(-1.0f0), st_f32(1.0f0)); order=st_ieee_order(), + created_by="parquet-cpp version 1.2.9") + @test ieeeold.trust.state == :untrusted + @test ieeeold.trust.reason == :parquet_cpp_pre_1_3 + counted = st_facts(rawleaf, st_modern(UInt8[0x01], UInt8[0x02]; + nulls=Int64(1)); created_by="parquet-cpp") + @test counted.lower.reason == :parquet_251 + @test counted.null_count.value == 1 + deprecated = st_facts(rawleaf, st_deprecated(UInt8[0x01], UInt8[0x02]); + created_by="parquet-cpp version 1.2.9") + @test deprecated.trust.reason == :parquet_cpp_pre_1_3 + @test deprecated.comparison == :undefined + @test deprecated.lower.reason == :deprecated_order_mismatch + + for createdby in ("parquet-cpp version 1.2.9", + "parquet-mr version 1.9.9") + modern = st_facts(rawleaf, distinct; created_by=createdby) + @test modern.comparison == :unsigned_bytes + @test modern.lower.reason in ( + :parquet_cpp_pre_1_3, :parquet_mr_pre_1_10) + deprecated = st_facts(rawleaf, + st_deprecated(UInt8[0x01], UInt8[0x02]); created_by=createdby) + @test deprecated.comparison == :undefined + @test deprecated.lower.reason == :deprecated_order_mismatch + end + empty = st_modern(UInt8[0x01], UInt8[0x02]; nulls=Int64(4)) + for createdby in ("parquet-cpp version 1.2.9", + "parquet-mr version 1.9.9"), order in (nothing, st_unknown_order()) + facts = st_facts(rawleaf, empty; created_by=createdby, order=order) + @test facts.occupancy == :no_non_null + @test facts.trust.state == :untrusted + @test facts.lower.reason == (order === nothing ? :missing_order : + :unknown_order) + end +end diff --git a/test/table.jl b/test/table.jl new file mode 100644 index 0000000..50b5813 --- /dev/null +++ b/test/table.jl @@ -0,0 +1,692 @@ +using Tables +using Dates + +function tablereplace(value; overrides...) + names = fieldnames(typeof(value)) + values = map(names) do name + return get(overrides, name, getproperty(value, name)) + end + return typeof(value)(values...) +end + +function tablerewritefooter(bytes::Vector{UInt8}, metadata) + file = Parquet.File(bytes) + prefix = collect(@view bytes[1:Int(file.footer.offset)]) + close(file) + footer = Parquet.Thrift.encode(metadata) + output = copy(prefix) + append!(output, footer) + Parquet._writelittle!(output, UInt32(length(footer))) + append!(output, Parquet.PARQUET_MAGIC) + return output +end + +function tableprematerialized(bytes::Vector{UInt8}; + limits::Parquet.Limits=Parquet.Limits()) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserveobjects!(budget, 2) + file = Parquet.File(bytes; limits=limits, budget=budget) + try + metadata = Parquet._readfilemetadata(file, limits, budget) + Parquet.Schema(metadata; limits=limits, budget=budget) + return Parquet._budgetused(budget) + finally + close(file) + end +end + +function tabledeepstruct(depth::Int) + depth >= 1 || throw(ArgumentError("deep table depth must be positive")) + values::AbstractVector = Int32[7] + for level in depth:-1:1 + child = level == depth ? "value" : "level_$(level + 1)" + values = Parquet.StructVector(String[child], AbstractVector[values]; + rows=1) + end + return values +end + +function tableintervalresult(bytes::Vector{UInt8}, limits::Parquet.Limits) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserveobjects!(budget, 2) + file = Parquet.File(bytes; limits=limits, budget=budget) + try + metadata = Parquet._readfilemetadata(file, limits, budget) + schema = Parquet.Schema(metadata; limits=limits, budget=budget) + baseline = Parquet._budgetused(budget) + preflight = Parquet._preflightoffsetindexdeclarations(file, metadata, + schema, limits) + result = try + Parquet._validatepageindexdeclarationoverlaps!(file, metadata, + schema, preflight, budget) + catch err + err + end + return result, preflight, baseline, Parquet._budgetused(budget) + finally + close(file) + end +end + +mutable struct TableCallbackSentinel <: Exception + id::Int +end + +mutable struct TableCallbackSource <: Parquet.AbstractSource + bytes::Vector{UInt8} + failread::Int + reads::Int + closes::Int + sentinel::TableCallbackSentinel +end + +mutable struct TableThrowingCloseSource <: Parquet.AbstractSource + bytes::Vector{UInt8} + failread::Int + reads::Int + closes::Int + readsentinel::TableCallbackSentinel + closesentinel::TableCallbackSentinel +end + +function Parquet.sourcelength(source::TableCallbackSource) + return Int64(length(source.bytes)) +end + +function Parquet.readrange(source::TableCallbackSource, offset::Integer, + count::Integer) + source.reads += 1 + source.reads == source.failread && throw(source.sentinel) + first = Int(offset) + 1 + return @view source.bytes[first:(first + Int(count) - 1)] +end + +function Parquet.close!(source::TableCallbackSource) + source.closes += 1 + return +end + +function Parquet.sourcelength(source::TableThrowingCloseSource) + return Int64(length(source.bytes)) +end + +function Parquet.readrange(source::TableThrowingCloseSource, offset::Integer, + count::Integer) + source.reads += 1 + source.reads == source.failread && throw(source.readsentinel) + first = Int(offset) + 1 + return @view source.bytes[first:(first + Int(count) - 1)] +end + +function Parquet.close!(source::TableThrowingCloseSource) + source.closes += 1 + throw(source.closesentinel) +end + +@testset "flat Tables facade" begin + input = ( + id=Int64[1, 2, 3], + flag=Bool[true, false, true], + score=Union{Missing,Float64}[1.5, missing, -2.0], + name=["alpha", "βeta", ""], + label=Union{Missing,String}["first", missing, "κ"], + ) + bytes = Parquet._encodefile(input) + table = Parquet.Table(bytes) + @test length(table) == 3 + @test Tables.istable(typeof(table)) + @test Tables.columnaccess(typeof(table)) + @test Tables.columnnames(table) == (:id, :flag, :score, :name, :label) + @test Tables.schema(table).types == (Int64, Bool, Union{Missing,Float64}, String, Union{Missing,String}) + columns = Tables.columntable(table) + @test columns.id == input.id + @test columns.flag == input.flag + @test isequal(columns.score, input.score) + @test columns.name == input.name + @test isequal(columns.label, input.label) + close(table) + close(table) +end + +@testset "Apache optional LIST fixture" begin + corpus = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + fixture = joinpath(corpus, "data", "list_columns.parquet") + if isfile(fixture) + table = Parquet.Table(fixture) + expectedintegers = Union{Missing,Vector{Union{Missing,Int64}}}[ + Union{Missing,Int64}[1, 2, 3], + Union{Missing,Int64}[missing, 1], + Union{Missing,Int64}[4], + ] + expectedstrings = Union{Missing,Vector{Union{Missing,String}}}[ + Union{Missing,String}["abc", "efg", "hij"], + missing, + Union{Missing,String}["efg", missing, "hij", "xyz"], + ] + @test isequal(table.columns.int64_list, expectedintegers) + @test isequal(table.columns.utf8_list, expectedstrings) + close(table) + else + @info "parquet-testing corpus not found; skipping LIST Tables fixture" corpus + end +end + +@testset "DATE and optional LIST Tables facade" begin + dates = Union{Missing,Date}[Date(1969, 12, 31), missing, Date(1970, 1, 1), + Date(2000, 2, 29)] + for pageversion in (:v1, :v2) + table = Parquet.Table(Parquet._encodefile((dates=dates,); + pageversion=pageversion, codec=:snappy)) + @test Tables.schema(table).types == (Union{Missing,Date},) + @test isequal(table.columns.dates, dates) + close(table) + end + + days = Union{Missing,Vector{Union{Missing,Date}}}[ + missing, + Union{Missing,Date}[], + Union{Missing,Date}[missing], + Union{Missing,Date}[Date(1970, 1, 1), missing, Date(1969, 12, 31)], + Union{Missing,Date}[Date(2000, 2, 29)], + ] + for pageversion in (:v1, :v2) + table = Parquet.Table(Parquet._encodefile((id=Int32[1, 2, 3, 4, 5], days=days); + pageversion=pageversion, codec=:snappy)) + @test Tables.schema(table).types == + (Int32, Union{Missing,Parquet.ListValue{Union{Missing,Date}}}) + @test table.columns.id == Int32[1, 2, 3, 4, 5] + @test isequal(table.columns.days, days) + close(table) + end +end + +@testset "flat Tables facade validation" begin + bytes = Parquet._encodefile((a=Int32[1, 2],)) + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(copy(file.footer.bytes), Parquet.Metadata.FileMetaData) + close(file) + badmetadata = Parquet.Metadata.FileMetaData( + version=metadata.version, + schema=metadata.schema, + num_rows=Int64(3), + row_groups=metadata.row_groups, + created_by=metadata.created_by, + ) + footer = Parquet.Thrift.encode(badmetadata) + bad = vcat(bytes[1:(Int(metadata.row_groups[1].total_byte_size) + 4)], footer, + reinterpret(UInt8, [htol(UInt32(length(footer)))]), Parquet.PARQUET_MAGIC) + @test_throws Parquet.FormatError Parquet.Table(bad) + @test_throws Parquet.LimitError Parquet.Table(bytes; limits=Parquet.Limits(max_container_elements=1)) + element = Parquet.Metadata.SchemaElement( + name="bad", + type_=Parquet.Metadata.Type.BYTE_ARRAY, + repetition_type=Parquet.Metadata.FieldRepetitionType.REQUIRED, + logicalType=Parquet.Metadata.LogicalType(STRING=Parquet.Metadata.StringType()), + ) + node = Parquet.SchemaNode(element, ["bad"], Int16(0), Int16(0), Int32(1), Parquet.SchemaNode[]) + @test_throws Parquet.FormatError Parquet._tablevalues(node, [UInt8[0xff]]) +end + +@testset "table name validation rollback and precedence" begin + required = Parquet.Metadata.FieldRepetitionType.REQUIRED + for (names, message) in ((("duplicate", "duplicate"), "unique"), + (("valid", "nul\0name"), "containing NUL")) + elements = Parquet.Metadata.SchemaElement[ + Parquet.Metadata.SchemaElement(name="root", num_children=Int32(2)), + Parquet.Metadata.SchemaElement(name=names[1], + type_=Parquet.Metadata.Type.INT32, repetition_type=required), + Parquet.Metadata.SchemaElement(name=names[2], + type_=Parquet.Metadata.Type.INT32, repetition_type=required), + ] + schema = Parquet.Schema(elements) + limits = Parquet.Limits(max_schema_name_bytes=0) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, Int64(64)) + before = Parquet._internedschemanamebytes() + error = try + Parquet._tablenames(schema, limits, budget) + nothing + catch err + err + end + @test error isa Parquet.UnsupportedFeatureError + @test occursin(message, error.message) + @test Parquet._budgetused(budget) == 64 + @test Parquet._internedschemanamebytes() == before + end + + unique = "__parquet_table_name_limit_rollback__" + elements = Parquet.Metadata.SchemaElement[ + Parquet.Metadata.SchemaElement(name="root", num_children=Int32(1)), + Parquet.Metadata.SchemaElement(name=unique, + type_=Parquet.Metadata.Type.INT32, repetition_type=required), + ] + schema = Parquet.Schema(elements) + limits = Parquet.Limits(max_schema_name_bytes=0) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, Int64(64)) + before = Parquet._internedschemanamebytes() + @test_throws Parquet.LimitError Parquet._tablenames(schema, limits, + budget) + @test Parquet._budgetused(budget) == 64 + @test Parquet._internedschemanamebytes() == before +end + +@testset "table metadata preflight precedes nested allocation" begin + bytes = Parquet._encodefile((value=Int32[1],)) + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, + Parquet.Metadata.FileMetaData) + footeroffset = file.footer.offset + close(file) + group = only(metadata.row_groups) + chunk = only(group.columns) + column = something(chunk.meta_data) + negativeindexmetadata = tablereplace(metadata; + row_groups=[tablereplace(group; + columns=[tablereplace(chunk; offset_index_offset=Int64(-1), + offset_index_length=Int32(1))])]) + sortingnegative = Parquet.Metadata.SortingColumn(column_idx=Int32(-1), + descending=false, nulls_first=false) + sortingpast = Parquet.Metadata.SortingColumn( + column_idx=Int32(length(group.columns)), descending=false, + nulls_first=false) + malformed = ( + tablereplace(metadata; row_groups=[tablereplace(group; + num_rows=Int64(-1))]), + tablereplace(metadata; row_groups=[tablereplace(group; + total_byte_size=Int64(-1))]), + tablereplace(metadata; row_groups=[tablereplace(group; + total_compressed_size=Int64(-1))]), + tablereplace(metadata; row_groups=[tablereplace(group; + file_offset=Int64(-1))]), + tablereplace(metadata; row_groups=[tablereplace(group; + file_offset=Int64(1))]), + tablereplace(metadata; row_groups=[tablereplace(group; + file_offset=footeroffset + 1)]), + tablereplace(metadata; row_groups=[tablereplace(group; + ordinal=Int16(-1))]), + tablereplace(metadata; row_groups=[tablereplace(group; + sorting_columns=[sortingnegative])]), + tablereplace(metadata; row_groups=[tablereplace(group; + sorting_columns=[sortingpast])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; file_offset=Int64(-1))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; meta_data=tablereplace(column; + total_compressed_size=Int64(-1)))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; meta_data=tablereplace(column; + data_page_offset=Int64(-1)))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; offset_index_length=nothing)])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; column_index_offset=Int64(4), + column_index_length=nothing)])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; offset_index_offset=nothing, + offset_index_length=nothing, column_index_offset=Int64(4), + column_index_length=Int32(1))])]), + negativeindexmetadata, + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; offset_index_length=Int32(0))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; + offset_index_offset=typemax(Int64), + offset_index_length=Int32(1))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; offset_index_offset=footeroffset, + offset_index_length=Int32(1))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; column_index_offset=Int64(-1), + column_index_length=Int32(1))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; column_index_offset=Int64(4), + column_index_length=Int32(0))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; + column_index_offset=typemax(Int64), + column_index_length=Int32(1))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; column_index_offset=footeroffset, + column_index_length=Int32(1))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; meta_data=tablereplace(column; + bloom_filter_offset=Int64(-1)))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; meta_data=tablereplace(column; + bloom_filter_length=Int32(1)))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; meta_data=tablereplace(column; + bloom_filter_offset=Int64(4), + bloom_filter_length=Int32(0)))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; meta_data=tablereplace(column; + bloom_filter_offset=typemax(Int64), + bloom_filter_length=Int32(1)))])]), + tablereplace(metadata; row_groups=[tablereplace(group; + columns=[tablereplace(chunk; meta_data=tablereplace(column; + bloom_filter_offset=footeroffset, + bloom_filter_length=Int32(1)))])]), + ) + messages = ("negative row count", "negative total byte size", + "negative compressed byte size", "negative file offset", + "outside the file body", "outside the file body", "negative ordinal", + "sorting column index", "sorting column index", "negative file offset", + "negative column chunk size", "negative data page offset", + "offset-index offset and length", "column-index offset and length", + "required offset index", "offset-index offset", "offset-index length", + "overflows Int64", "extends past the footer", "column-index offset", + "column-index length", "overflows Int64", "extends past the footer", + "bloom-filter offset", + "bloom-filter length is present without its offset", + "bloom-filter length must be positive", "overflows Int64", + "extends past the footer") + for (hostilemetadata, message) in zip(malformed, messages) + hostile = tablerewritefooter(bytes, hostilemetadata) + maximum = tableprematerialized(hostile) + error = try + Parquet.Table(hostile; limits=Parquet.Limits( + max_materialized_bytes=maximum)) + nothing + catch err + err + end + @test error isa Parquet.FormatError + @test occursin(message, sprint(showerror, error)) + end + + maximum = tableprematerialized(bytes) + pageindexerror = try + Parquet.Table(bytes; limits=Parquet.Limits( + max_materialized_bytes=maximum, + max_page_index_bytes=Int64(something( + chunk.offset_index_length)) - 1)) + nothing + catch err + err + end + @test pageindexerror isa Parquet.LimitError + @test pageindexerror.resource == :page_index_bytes + + hostile = tablerewritefooter(bytes, negativeindexmetadata) + source = TableCallbackSource(hostile, 0, 0, 0, + TableCallbackSentinel(5)) + sourceerror = try + Parquet.Table(source; limits=Parquet.Limits( + max_materialized_bytes=tableprematerialized(hostile))) + nothing + catch err + err + end + @test sourceerror isa Parquet.FormatError + @test source.reads == 3 + @test source.closes == 1 + + legacychunk = tablereplace(chunk; offset_index_offset=nothing, + offset_index_length=nothing, meta_data=tablereplace(column; + bloom_filter_offset=Int64(something(chunk.offset_index_offset)), + bloom_filter_length=nothing)) + legacybloom = tablereplace(metadata; + row_groups=[tablereplace(group; columns=[legacychunk])]) + legacytable = Parquet.Table(tablerewritefooter(bytes, legacybloom)) + @test legacytable.columns.value == Int32[1] + close(legacytable) + + zerogroup = tablereplace(metadata; + row_groups=[tablereplace(group; file_offset=Int64(0))]) + zerotable = Parquet.Table(tablerewritefooter(bytes, zerogroup)) + @test zerotable.columns.value == Int32[1] + close(zerotable) + + twobytes = Parquet._encodefile((left=Int32[1], right=Int32[2])) + twofile = Parquet.File(twobytes) + twometadata = Parquet.Thrift.decode(twofile.footer.bytes, + Parquet.Metadata.FileMetaData) + close(twofile) + twogroup = only(twometadata.row_groups) + firstchunk, secondchunk = twogroup.columns + firstcolumn = something(firstchunk.meta_data) + secondcolumn = something(secondchunk.meta_data) + for bloomlength in (Int32(1), nothing) + bloomchunk = tablereplace(secondchunk; + meta_data=tablereplace(secondcolumn; + bloom_filter_offset=firstcolumn.data_page_offset, + bloom_filter_length=bloomlength)) + bloommetadata = tablereplace(twometadata; + row_groups=[tablereplace(twogroup; + columns=[firstchunk, bloomchunk])]) + bloomhostile = tablerewritefooter(twobytes, bloommetadata) + generous = Parquet.Limits(max_materialized_bytes=1_000_000_000) + direct, bloompreflight, bloombaseline, bloomafter = + tableintervalresult(bloomhostile, generous) + @test direct isa Parquet.FormatError + @test bloombaseline == bloomafter + bloomscratch = Parquet._materializedarraybytes( + Parquet._PageIndexInterval, bloompreflight.intervalcount) + bloomerror = try + Parquet.Table(bloomhostile; limits=Parquet.Limits( + max_materialized_bytes=bloombaseline + bloomscratch)) + nothing + catch err + err + end + @test bloomerror isa Parquet.FormatError + @test occursin("storage ranges overlap", sprint(showerror, bloomerror)) + end +end + +@testset "deep page-index overlap preflight and scratch rollback" begin + depth = 256 + wide = Parquet.Limits(max_metadata_depth=depth + 4, + max_materialized_bytes=1_000_000_000) + bytes = Parquet._encodefile((deep=tabledeepstruct(depth),); limits=wide) + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, + Parquet.Metadata.FileMetaData) + close(file) + group = only(metadata.row_groups) + chunk = only(group.columns) + column = something(chunk.meta_data) + overlapchunk = tablereplace(chunk; + offset_index_offset=column.data_page_offset, + offset_index_length=chunk.offset_index_length) + hostilemetadata = tablereplace(metadata; + row_groups=[tablereplace(group; columns=[overlapchunk])]) + hostile = tablerewritefooter(bytes, hostilemetadata) + baselimits = Parquet.Limits(max_metadata_depth=depth + 4, + max_materialized_bytes=1_000_000_000) + baseline = tableprematerialized(hostile; limits=baselimits) + scratch = Parquet._materializedarraybytes(Parquet._PageIndexInterval, 2) + + exactlimits = Parquet.Limits(max_metadata_depth=depth + 4, + max_materialized_bytes=baseline + scratch) + exact, preflight, exactbaseline, exactafter = + tableintervalresult(hostile, exactlimits) + @test preflight.intervalcount == 2 + @test !preflight.overlaps_validated + @test exact isa Parquet.FormatError + @test occursin("storage ranges overlap", sprint(showerror, exact)) + @test exactbaseline == exactafter == baseline + + lowlimits = Parquet.Limits(max_metadata_depth=depth + 4, + max_materialized_bytes=baseline + scratch - 1) + low, _, lowbaseline, lowafter = tableintervalresult(hostile, lowlimits) + @test low isa Parquet.LimitError + @test low.resource == :materialized_bytes + @test lowbaseline == lowafter == baseline + + valid, _, validbaseline, validafter = tableintervalresult(bytes, + exactlimits) + @test valid isa Parquet._PageIndexRangePreflight + @test valid.overlaps_validated + @test validbaseline == validafter + + tableerror = try + Parquet.Table(hostile; limits=exactlimits) + nothing + catch err + err + end + @test tableerror isa Parquet.FormatError + @test occursin("storage ranges overlap", sprint(showerror, tableerror)) +end + +@testset "Table source exception identity and ownership" begin + bytes = Parquet._encodefile((value=Int32[1],)) + failedsentinel = TableCallbackSentinel(1) + failed = TableCallbackSource(bytes, 1, 0, 0, failedsentinel) + failederror = try + Parquet.Table(failed) + nothing + catch err + err + end + @test failederror === failedsentinel + @test failed.reads == 1 + @test failed.closes == 0 + + adoptedsentinel = TableCallbackSentinel(2) + adopted = TableCallbackSource(bytes, 4, 0, 0, adoptedsentinel) + adoptederror = try + Parquet.Table(adopted) + nothing + catch err + err + end + @test adoptederror === adoptedsentinel + @test adopted.reads == 4 + @test adopted.closes == 1 + + successful = TableCallbackSource(bytes, 0, 0, 0, + TableCallbackSentinel(3)) + table = Parquet.Table(successful) + @test table.columns.value == Int32[1] + @test successful.closes == 0 + close(table) + @test successful.closes == 1 + + readsentinel = TableCallbackSentinel(4) + closesentinel = TableCallbackSentinel(5) + throwingclose = TableThrowingCloseSource(bytes, 4, 0, 0, readsentinel, + closesentinel) + throwingerror = try + Parquet.Table(throwingclose) + nothing + catch err + err + end + @test throwingerror === readsentinel + @test throwingclose.reads == 4 + @test throwingclose.closes == 1 +end + +@testset "fixed-width Tables schema preservation" begin + required = NTuple{3,UInt8}[(0x01, 0x02, 0x03), (0x04, 0x05, 0x06)] + optional = Union{Missing,NTuple{2,UInt8}}[(0x07, 0x08), missing] + input = (required=required, optional=optional) + expected_required = Vector{UInt8}[UInt8[1, 2, 3], UInt8[4, 5, 6]] + expected_optional = Union{Missing,Vector{UInt8}}[UInt8[7, 8], missing] + table = Parquet.Table(Parquet._encodefile(input)) + @test table.columns.required == expected_required + @test isequal(table.columns.optional, expected_optional) + @test table.columns.required isa Parquet.FixedByteArrayVector + @test table.columns.required.width == 3 + @test table.columns.optional.width == 2 + @test collect(table.columns.required) == expected_required + @test copy(table.columns.required) == expected_required + table.columns.required[1] = UInt8[9, 9, 9] + @test table.columns.required[1] == UInt8[9, 9, 9] + table.columns.required[1] = expected_required[1] + @test_throws ArgumentError setindex!(table.columns.required, UInt8[1], 1) + @test Tables.schema(table).types == + (Vector{UInt8}, Union{Missing,Vector{UInt8}}) + rewritten = Parquet._encodefile(table) + close(table) + + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + @test [element.type_ for element in metadata.schema[2:end]] == + fill(Parquet.Metadata.Type.FIXED_LEN_BYTE_ARRAY, 2) + @test [element.type_length for element in metadata.schema[2:end]] == [3, 2] + close(file) + table = Parquet.Table(rewritten) + @test table.columns.required == expected_required + @test isequal(table.columns.optional, expected_optional) + close(table) + + short = Parquet.Table(Parquet._encodefile(input)) + pop!(short.columns.required[1]) + @test_throws ArgumentError Parquet._encodefile(short) + close(short) + long = Parquet.Table(Parquet._encodefile(input)) + push!(long.columns.required[1], 0xff) + @test_throws ArgumentError Parquet._encodefile(long) + close(long) +end + +@testset "fixed-width Tables schema limit" begin + bytes = Parquet._encodefile((value=NTuple{1,UInt8}[],)) + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + prefix = bytes[1:Int(file.footer.offset)] + close(file) + schema = copy(metadata.schema) + element = schema[2] + schema[2] = Parquet.Metadata.SchemaElement( + type_=element.type_, + type_length=typemax(Int32), + repetition_type=element.repetition_type, + name=element.name, + ) + hostile = Parquet.Metadata.FileMetaData( + version=metadata.version, + schema=schema, + num_rows=metadata.num_rows, + row_groups=metadata.row_groups, + created_by=metadata.created_by, + ) + footer = Parquet.Thrift.encode(hostile) + trailer = UInt8[] + Parquet._writelittle!(trailer, UInt32(length(footer))) + append!(trailer, Parquet.PARQUET_MAGIC) + input = vcat(prefix, footer, trailer) + error = try + Parquet.Table(input; limits=Parquet.Limits(max_string_bytes=64)) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :string_bytes + @test error.requested == typemax(Int32) + @test error.maximum == 64 +end + +@testset "flat Tables corpus facade" begin + corpus = get(ENV, "PARQUET_TESTING_DIR", joinpath(@__DIR__, "parquet-testing")) + fixture = joinpath(corpus, "data", "datapage_v1-uncompressed-checksum.parquet") + corrupt = joinpath(corpus, "data", "datapage_v1-corrupt-checksum.parquet") + if isfile(fixture) + table = Parquet.Table(fixture) + columns = Tables.columntable(table) + @test keys(columns) == (:a, :b) + @test length(table) == 5120 + @test sum(Int64, columns.a) == 43118090240 + @test sum(Int64, columns.b) == 129016125440 + close(table) + @test_throws Parquet.FormatError Parquet.Table(corrupt) + encrypted = joinpath(corpus, "data", "encrypt_columns_plaintext_footer.parquet.encrypted") + if isfile(encrypted) + error = try + Parquet.Table(encrypted) + catch err + err + end + @test error isa Parquet.FormatError + @test occursin("plaintext-footer encryption", sprint(showerror, error)) + end + else + @info "parquet-testing corpus not found; skipping Tables corpus facade" corpus + end +end diff --git a/test/test_codec.jl b/test/test_codec.jl deleted file mode 100644 index c71a422..0000000 --- a/test/test_codec.jl +++ /dev/null @@ -1,170 +0,0 @@ -using Parquet -using Decimals -using Test - -const decimal_encoding_testdata = [ - ( - precision=Int32(19), - scale=Int32(0), - datatype=Int128, - byte_data=[ - UInt8[0,0,0,0,1], - UInt8[0,0,0,1,1], - UInt8[0,0,1,1,1], - UInt8[0,1,1,1,1], - UInt8[1,1,1,1,1], - ], - converted_data=Int128[1, 257, 65793, 16843009, 4311810305] - ), - ( - precision=Int32(10), - scale=Int32(0), - datatype=Int64, - byte_data=[ - UInt8[0,0,0,0,1], - UInt8[0,0,0,1,1], - UInt8[0,0,1,1,1], - UInt8[0,1,1,1,1], - UInt8[1,1,1,1,1], - ], - converted_data=Int64[1, 257, 65793, 16843009, 4311810305] - ), - ( - precision=Int32(5), - scale=Int32(0), - datatype=Int32, - byte_data=[ - UInt8[0,0,0,1], - UInt8[0,0,1,1], - UInt8[0,1,1,1], - ], - converted_data=Int32[1, 257, 65793] - ), - ( - precision=Int32(4), - scale=Int32(0), - datatype=Int16, - byte_data=[ - UInt8[0,1], - UInt8[1,1], - ], - converted_data=Int16[1, 257] - ), - ( - precision=Int32(4), - scale=Int32(2), - datatype=Decimal, - byte_data=[ - UInt8[0,1], - UInt8[1,1], - ], - converted_data=Decimal[Decimal(0, 1, -2), Decimal(0, 257, -2)] - ), - ( - precision=Int32(19), - scale=Int32(5), - datatype=Decimal, - byte_data=[ - UInt8[0,0,0,0,1], - UInt8[1,1,1,1,1], - ], - converted_data=Decimal[Decimal(0, 1, -5), Decimal(0, 4311810305, -5)] - ), - ( - precision=Int32(10), - scale=Int32(-2), - datatype=Decimal, - byte_data=[ - UInt8[0,0,0,0,1], - UInt8[0,0,0,1,1], - ], - converted_data=[Decimal(0, 1, 2), Decimal(0, 257, 2)] - ), - ( - precision=Int32(4), - scale=Int32(2), - datatype=Decimal, - byte_data=Int64[ - 200, - 1234, - ], - converted_data=Decimal[Decimal(0, 200, -2), Decimal(0, 1234, -2)] - ), - ( - precision=Int32(4), - scale=Int32(2), - datatype=Decimal, - byte_data=Int32[ - 200, - 1234, - ], - converted_data=Decimal[Decimal(0, 200, -2), Decimal(0, 1234, -2)] - ), -] - -function test_codec() - @debug("testing reading bitpacked run (old scheme)") - let data = UInt8[0x05, 0x39, 0x77] - byte_width = Parquet.@bit2bytewidth(UInt8(3)) - typ = Parquet.@byt2itype(byte_width) - #arr = Array{typ}(undef, 8) - inp = Parquet.InputState(data, 0) - out = Parquet.OutputState(typ, 8) - Parquet.read_bitpacked_run_old(inp, out, Int32(8), UInt8(3)) - @test out.data == Int32[0:7;] - end - - @debug("testing reading bitpacked run") - let data = UInt8[0x88, 0xc6, 0xfa] - byte_width = Parquet.@bit2bytewidth(UInt8(3)) - typ = Parquet.@byt2itype(byte_width) - #arr = Array{typ}(undef, 8) - inp = Parquet.InputState(data, 0) - out = Parquet.OutputState(typ, 8) - Parquet.read_bitpacked_run(inp, out, 8, UInt8(3), byte_width) - @test out.data == Int32[0:7;] - end - - @debug("testing decimal decoding") - for data in decimal_encoding_testdata - (d, f1) = Parquet.map_logical_decimal(Int32(data.precision), Int32(data.scale)) - f2 = (bytes)->Parquet.logical_decimal(bytes, data.precision, data.scale) - f3 = (bytes)->Parquet.logical_decimal(bytes, data.precision, data.scale; use_float=true) - @test d === data.datatype - if isbitstype(d) - @test all(map(f1, data.byte_data) .=== data.converted_data) - @test all(map(f2, data.byte_data) .=== data.converted_data) - @test all(map(f3, data.byte_data) .=== data.converted_data) - else - @test all(map(f1, data.byte_data) .== data.converted_data) - @test all(map(f2, data.byte_data) .== data.converted_data) - @test all(map(f3, data.byte_data) .== convert(Vector{Float64}, data.converted_data)) - end - end - - iob = IOBuffer() - write(iob, Int64(1000)) - write(iob, Int64(1234)) - is = Parquet.InputState(take!(iob), 0) - os = Parquet.OutputState(Decimals.Decimal, 2) - Parquet.read_plain_values(is, os, Int32(2), (v)->Parquet.logical_decimal(v, 4, 2), Parquet.PAR2._Type.INT64) - @test os.data == Decimal[Decimal(0, 1000, -2), Decimal(0, 1234, -2)] - - iob = IOBuffer() - write(iob, Int32(1000)) - write(iob, Int32(1234)) - is = Parquet.InputState(take!(iob), 0) - os = Parquet.OutputState(Decimals.Decimal, 2) - Parquet.read_plain_values(is, os, Int32(2), (v)->Parquet.logical_decimal(v, 4, 2), Parquet.PAR2._Type.INT32) - @test os.data == Decimal[Decimal(0, 1000, -2), Decimal(0, 1234, -2)] - - iob = IOBuffer(UInt8[0,0,0,0,1,0,0,0,1,1]) - is = Parquet.InputState(take!(iob), 0) - os = Parquet.OutputState(Decimals.Decimal, 2) - Parquet.read_plain_values(is, os, Int32(2), (v)->Parquet.logical_decimal(v, 4, 2), Parquet.PAR2._Type.FIXED_LEN_BYTE_ARRAY) - @test os.data == Decimal[Decimal(0, 1, -2), Decimal(0, 257, -2)] -end - -@testset "codec" begin - test_codec() -end diff --git a/test/test_cursors.jl b/test/test_cursors.jl deleted file mode 100644 index 7c95c07..0000000 --- a/test/test_cursors.jl +++ /dev/null @@ -1,78 +0,0 @@ -using Parquet -using Test - -function test_row_cursor(file::String) - p = Parquet.File(file) - - t1 = time() - nr = nrows(p) - cnames = colnames(p) - rc = RecordCursor(p) - rec = nothing - nread = 0 - for i in rc - rec = i - nread += 1 - end - @test nr == nread - @debug("loaded", file, count=nr, last_record=rec, time_to_read=time()-t1) - - iob = IOBuffer() - show(iob, rc) - sb = take!(iob) - @test !isempty(sb) - @debug("row cursor show", file, showbuffer=String(sb)) -end - -function test_batchedcols_cursor(file::String) - p = Parquet.File(file) - - t1 = time() - nr = nrows(p) - cnames = colnames(p) - cc = BatchedColumnsCursor(p) - batch = nothing - nread = 0 - for i in cc - batch = i - nread += length(first(batch)) - end - @test nr == nread - @debug("loaded", file, count=nr, ncols=length(propertynames(batch)), time_to_read=time()-t1) - - iob = IOBuffer() - show(iob, cc) - sb = take!(iob) - @test !isempty(sb) - @debug("batched column cursor show", file, showbuffer=String(sb)) -end - -function test_row_cursor_all_files() - for encformat in ("SNAPPY", "GZIP", "NONE") - for fname in ("nation", "customer") - test_row_cursor(joinpath(parcompat, "parquet-testdata", "impala", "1.1.1-$encformat/$fname.impala.parquet")) - end - end -end - -function test_batchedcols_cursor_all_files() - for encformat in ("SNAPPY", "GZIP", "NONE") - for fname in ("nation", "customer") - test_batchedcols_cursor(joinpath(parcompat, "parquet-testdata", "impala", "1.1.1-$encformat/$fname.impala.parquet")) - end - end -end - -function test_col_cursor_length() - path = joinpath(parcompat, "parquet-testdata", "impala", "1.1.1-SNAPPY/nation.impala.parquet") - pq_file = Parquet.File(path) - col_name = pq_file |> colnames |> first - col_cursor = Parquet.ColCursor(pq_file, col_name) - @test length(col_cursor) == 25 -end - -@testset "cursors" begin - test_row_cursor_all_files() - test_batchedcols_cursor_all_files() - test_col_cursor_length() -end diff --git a/test/test_load.jl b/test/test_load.jl deleted file mode 100644 index 3cb1471..0000000 --- a/test/test_load.jl +++ /dev/null @@ -1,464 +0,0 @@ -using Parquet -using Test -using Dates -using Tables - -function test_load(file::String) - p = Parquet.File(file) - @debug("load file", file) - @test isa(p.meta, Parquet.FileMetaData) - - rgs = rowgroups(p) - @test length(rgs) > 0 - - cnames = colnames(p) - @test length(cnames) > 0 - @debug("data columns", file, cnames) - - for rg in rgs - ccs = columns(p, rg) - @debug("reading row group", file, ncolumnchunks=length(ccs)) - - for cc in ccs - npages = 0 - ccp = Parquet.ColumnChunkPages(p, cc) - result = iterate(ccp) - npages = 0 - ncompressedbytes = 0 - nuncompressedbytes = 0 - while result !== nothing - page,nextpos = result - result = iterate(ccp, nextpos) - npages += 1 - ncompressedbytes += Parquet.page_size(page.hdr) - nuncompressedbytes += page.hdr.uncompressed_page_size - end - @test npages > 0 - @test ncompressedbytes > 0 - @test nuncompressedbytes >= 0 - @debug("read column chunk", file, npages, ncompressedbytes, nuncompressedbytes) - end - end - - @debug("done loading file", file, showbuffer=String(sb)) - - iob = IOBuffer() - show(iob, p) - sb = take!(iob) - @test !isempty(sb) - @debug("parquet file show", file, showbuffer=String(sb)) - - iob = IOBuffer() - show(iob, p.meta) - sb = take!(iob) - @test !isempty(sb) - @debug("parquet file metadata show", file, showbuffer=String(sb)) - - show(iob, schema(p)) - sb = take!(iob) - @test !isempty(sb) - @debug("schema show", file, showbuffer=String(sb)) - - nothing -end - -function test_decode(file) - p = Parquet.File(file) - @debug("load file", file) - @test isa(p.meta, Parquet.FileMetaData) - - rgs = rowgroups(p) - @test length(rgs) > 0 - - for rg in rgs - ccs = columns(p, rg) - @debug("reading row group", file, ncolumnchunks=length(ccs)) - - for cc in ccs - jtype = Parquet.elemtype(Parquet.elem(schema(p), colname(p,cc))) - npages = 0 - ccpv = Parquet.ColumnChunkPageValues(p, cc, jtype) - result = iterate(ccpv) - valcount = repncount = defncount = 0 - while result !== nothing - resultdata,nextpos = result - valcount += resultdata.value.offset - repncount += resultdata.repn_level.offset - defncount += resultdata.defn_level.offset - result = iterate(ccpv, nextpos) - npages += 1 - end - - @debug("read", file, npages, valcount, defncount, repncount) - end - end - nothing -end - -function test_decode_all_pages() - @testset "decode parquet-compatibility test files" begin - testfolder = parcompat - for encformat in ("SNAPPY", "GZIP", "NONE") - for fname in ("nation", "customer") - testfile = joinpath(testfolder, "parquet-testdata", "impala", "1.1.1-$encformat", "$fname.impala.parquet") - test_decode(testfile) - end - end - end - - @testset "decode julia-parquet-compatibility test files" begin - testfolder = julia_parcompat - for encformat in ("ZSTD", "SNAPPY", "GZIP", "NONE") - for fname in ("nation", "customer") - testfile = joinpath(testfolder, "Parquet_Files", "$(encformat)_pandas_pyarrow_$(fname).parquet") - test_decode(testfile) - end - end - end - - @testset "decode missingvalues test file" begin - test_decode(joinpath(@__DIR__, "missingvalues", "synthetic_data.parquet")) - end -end - -function test_load_all_pages() - @testset "load parquet-compatibility test files" begin - testfolder = parcompat - for encformat in ("SNAPPY", "GZIP", "NONE") - for fname in ("nation", "customer") - testfile = joinpath(testfolder, "parquet-testdata", "impala", "1.1.1-$encformat", "$fname.impala.parquet") - test_load(testfile) - end - end - end - - @testset "load julia-parquet-compatibility test files" begin - testfolder = julia_parcompat - for encformat in ("ZSTD", "SNAPPY", "GZIP", "NONE") - for fname in ("nation", "customer") - testfile = joinpath(testfolder, "Parquet_Files", "$(encformat)_pandas_pyarrow_$(fname).parquet") - test_load(testfile) - end - end - end -end - -function test_load_boolean_and_ts() - @testset "load booleans and timestamps" begin - @debug("load booleans and timestamps") - p = Parquet.File(joinpath(@__DIR__, "booltest", "alltypes_plain.snappy.parquet")) - - rg = rowgroups(p) - @test length(rg) == 1 - cc = columns(p, 1) - @test length(cc) == 11 - cnames = colnames(p) - @test length(cnames) == length(cc) - @test cnames[2] == ["bool_col"] - - rc = RecordCursor(p; rows=1:2, colnames=colnames(p)) - @test length(rc) == 2 - @test eltype(rc) == NamedTuple{(:id, :bool_col, :tinyint_col, :smallint_col, :int_col, :bigint_col, :float_col, :double_col, :date_string_col, :string_col, :timestamp_col),Tuple{Union{Missing, Int32},Union{Missing, Bool},Union{Missing, Int32},Union{Missing, Int32},Union{Missing, Int32},Union{Missing, Int64},Union{Missing, Float32},Union{Missing, Float64},Union{Missing, Array{UInt8,1}},Union{Missing, Array{UInt8,1}},Union{Missing, DateTime}}} - - values = collect(rc) - @test [v.bool_col for v in values] == [true,false] - @test [v.timestamp_col for v in values] == [DateTime("2009-04-01T12:00:00"), DateTime("2009-04-01T12:01:00")] - - cc = BatchedColumnsCursor(p) - values, _state = iterate(cc) - @test values.timestamp_col == [DateTime("2009-04-01T12:00:00"), DateTime("2009-04-01T12:01:00")] - - p = Parquet.File(joinpath(@__DIR__, "booltest", "alltypes_plain.snappy.parquet"); map_logical_types=Dict(["date_string_col"]=>(String,logical_string))) - rc = RecordCursor(p; rows=1:2, colnames=colnames(p)) - values = collect(rc) - @test [v.date_string_col for v in values] == ["04/01/09", "04/01/09"] - - cc = BatchedColumnsCursor(p) - values, _state = iterate(cc) - @test values.date_string_col == ["04/01/09", "04/01/09"] - - p = Parquet.File(joinpath(@__DIR__, "booltest", "alltypes_plain.snappy.parquet"); map_logical_types=Dict(["timestamp_col"]=>(DateTime,(v)->logical_timestamp(v; offset=Dates.Second(30))))) - rc = RecordCursor(p; rows=1:2, colnames=colnames(p)) - values = collect(rc) - @test [v.timestamp_col for v in values] == [DateTime("2009-04-01T12:00:30"), DateTime("2009-04-01T12:01:30")] - - cc = BatchedColumnsCursor(p) - values, _state = iterate(cc) - @test values.timestamp_col == [DateTime("2009-04-01T12:00:30"), DateTime("2009-04-01T12:01:30")] - #dlm,headers=readdlm("booltest/alltypes.csv", ','; header=true) - #@test [v.bool_col for v in values] == dlm[:,2] # skipping for now as this needs additional dependency on DelimitedFiles - end -end - -function test_load_nested() - @testset "load nested columns" begin - @debug("load nested columns") - p = Parquet.File(joinpath(@__DIR__, "nested", "nested1.parquet")) - - @test nrows(p) == 100 - @test ncols(p) == 5 - - rc = RecordCursor(p) - @test length(rc) == 100 - @test eltype(rc) == NamedTuple{(:_adobe_corpnew,),Tuple{NamedTuple{(:id, :vocab, :frequency, :max_len, :reduced_max_len),Tuple{Union{Missing, Int32},Union{Missing, String},Union{Missing, Int32},Union{Missing, Float64},Union{Missing, Int32}}}}} - - values = Any[] - for rec in rc - push!(values, rec) - end - - v = values[1]._adobe_corpnew - @test v.frequency == 3 - @test v.id == 1375 - @test v.max_len == 64192.0 - @test v.reduced_max_len == 64 - @test v.vocab == "10385911_a" - - v = values[100]._adobe_corpnew - @test v.frequency == 61322 - @test v.id == 724 - @test v.max_len == 64192.0 - @test v.reduced_max_len == 64 - @test v.vocab == "12400277_a" - - p = Parquet.File(joinpath(@__DIR__, "nested", "nested.parq")) - - @test nrows(p) == 10 - @test ncols(p) == 1 - - rc = RecordCursor(p) - @test length(rc) == 10 - @test eltype(rc) == NamedTuple{(:nest,),Tuple{Union{Missing, NamedTuple{(:thing,),Tuple{Union{Missing, NamedTuple{(:list,),Tuple{Array{NamedTuple{(:element,),Tuple{Union{Missing, String}}},1}}}}}}}}} - - values = collect(rc) - v = first(values) - @test length(v.nest.thing.list) == 2 - @test v.nest.thing.list[1].element == "hi" - v = last(values) - @test length(v.nest.thing.list) == 2 - @test v.nest.thing.list[1].element == "world" - end -end - -function test_load_multiple_rowgroups() - @testset "testing multiple rowgroups..." begin - @debug("load multiple rowgroups") - p = Parquet.File(joinpath(@__DIR__, "rowgroups", "multiple_rowgroups.parquet")) - - @test nrows(p) == 100 - @test ncols(p) == 12 - - rc = RecordCursor(p) - @test length(rc) == 100 - vals = collect(rc) - @test length(vals) == 100 - @test vals[1].int64 == vals[51].int64 - @test vals[1].int32 == vals[51].int32 - - cc = BatchedColumnsCursor(p) - @test length(cc) == 2 - colvals = collect(cc) - @test length(colvals) == 2 - @test length(colvals[1].int32) == 50 - end -end - -function test_load_file() - @testset "load a file" begin - filename = joinpath(@__DIR__, "rowgroups", "multiple_rowgroups.parquet") - - table = read_parquet(filename) - @test Tables.istable(table) - @test Tables.columnaccess(table) - @test Tables.schema(table).names == (:int32, :int64, :float32, :float64, :bool, :string, :int32m, :int64m, :float32m, :float64m, :boolm, :stringm) - cols = Tables.columns(table) - @test all([length(col)==100 for col in cols]) # all columns must be 100 rows long - @test length(cols) == 12 # 12 columns - partitions = Tables.partitions(table) - @test length(partitions) == 2 - partition_tables = collect(partitions) - @test length(partition_tables) == 2 - @test Tables.istable(partition_tables[1]) - @test Tables.columnaccess(partition_tables[1]) - close(table) - - table = read_parquet(filename; rows=1:10) - cols = Tables.columns(table) - @test all([length(col)==10 for col in cols]) # all columns must be 100 rows long - @test length(cols) == 12 # 12 columns - partitions = Tables.partitions(table) - @test length(partitions) == 1 - @test length(collect(partitions)) == 1 - close(table) - - table = read_parquet(filename; rows=1:100, batchsize=10) - cols = Tables.columns(table) - @test all([length(col)==100 for col in cols]) # all columns must be 100 rows long - @test length(cols) == 12 # 12 columns - partitions = Tables.partitions(table) - @test length(partitions) == 10 - @test length(collect(partitions)) == 10 - - iob = IOBuffer() - show(iob, table) - @test startswith(String(take!(iob)), "Parquet.Table(") - close(table) - - # test loading table with separately specified schema - parfile = joinpath(@__DIR__, "datasets", "bool_partition", "bool=False", "560bea059bf94bae9f785f9b4a455317.parquet") - schemafile = joinpath(@__DIR__, "datasets", "bool_partition", "_common_metadata") - schema = Tables.schema(read_parquet(schemafile)) - table = Parquet.Table(parfile, schema) - cols = Tables.columns(table) - @test length(cols) == 12 - @test all(ismissing.(cols.bool)) - close(table) - - # test loading table with custom missing partition column generator - table = Parquet.Table(parfile, schema; column_generator=(t,c,l)->trues(l)) - cols = Tables.columns(table) - @test sum(cols.bool) == 42 - end -end - -function test_load_at_offset() - @testset "load file at offset" begin - testfolder = parcompat - testfile = joinpath(testfolder, "parquet-testdata", "impala", "1.1.1-NONE", "customer.impala.parquet") - parquet_file = Parquet.File(testfile) - - vals_20000_40000 = first(collect(Parquet.BatchedColumnsCursor(parquet_file; rows=20000:40000))).c_custkey - vals_1_40000 = first(collect(Parquet.BatchedColumnsCursor(parquet_file; rows=1:40000))).c_custkey - vals_2_40001 = first(collect(Parquet.BatchedColumnsCursor(parquet_file; rows=2:40001))).c_custkey - - @test vals_20000_40000 == vals_1_40000[20000:40000] - @test vals_20000_40000 != vals_1_40000[1:20000] - @test vals_2_40001[1:10000] == vals_1_40000[2:10001] - @test vals_2_40001[1:10000] != vals_1_40000[1:10000] - end -end - -function test_mmap_mode() - @testset "memory map modes" begin - default_mode = Parquet._use_mmap[] - for mode in (true, false) - (mode === default_mode) && continue - Parquet.use_mmap(mode) - table = read_parquet(joinpath(@__DIR__, "rowgroups", "multiple_rowgroups.parquet")) - cols = Tables.columns(table) - @test all([length(col)==100 for col in cols]) - @test length(cols) == 12 - close(table) - end - Parquet.use_mmap(default_mode) - end -end - -function test_zero_rows() - @testset "load file with no rows" begin - table = read_parquet(joinpath(@__DIR__, "empty", "empty.parquet")) - cols = Tables.columns(table) - @test all([length(col)==0 for col in cols]) - @test length(cols) == 31 - partitions = Tables.partitions(table) - @test length(partitions) == 0 - schema = Tables.schema(table) - @test first(schema.types) == Union{Missing, Int64} - @test last(schema.types) == Union{Missing, String} - end -end - -function test_dataset() - @testset "load dataset" begin - # load dataset partitioned by boolean column - dataset_path = joinpath(@__DIR__, "datasets", "bool_partition") - - dataset = read_parquet(dataset_path) - @test Tables.istable(dataset) - @test Tables.columnaccess(dataset) - @test Tables.schema(dataset).names == (:int32, :int64, :float32, :float64, :bool, :string, :int32m, :int64m, :float32m, :float64m, :boolm, :stringm) - cols = Tables.columns(dataset) - @test all([length(col)==100 for col in cols]) # all columns must be 100 rows long - @test length(cols) == 12 # 12 columns - @test sum(cols.bool) == 58 # 58 rows in `bool=true` partition - - partitions = [] - for partition in Tables.partitions(dataset) - push!(partitions, partition) - end - @test length(partitions) == 2 - - iob = IOBuffer() - show(iob, dataset) - @test startswith(String(take!(iob)), "Parquet.Dataset(") - close(dataset) - - # load dataset with a filter - dataset = read_parquet(dataset_path; filter=(path)->occursin("bool=false", lowercase(path))) - @test Tables.istable(dataset) - @test Tables.columnaccess(dataset) - @test Tables.schema(dataset).names == (:int32, :int64, :float32, :float64, :bool, :string, :int32m, :int64m, :float32m, :float64m, :boolm, :stringm) - cols = Tables.columns(dataset) - @test all([length(col)==42 for col in cols]) # all bool=false columns must be 42 rows long - @test length(cols) == 12 # 12 columns - - partitions = [] - for partition in Tables.partitions(dataset) - push!(partitions, partition) - end - @test length(partitions) == 1 - close(dataset) - - # load dataset partitioned by string column - dataset_path = joinpath(@__DIR__, "datasets", "string_partition") - dataset = read_parquet(dataset_path) - @test Tables.istable(dataset) - @test Tables.columnaccess(dataset) - @test Tables.schema(dataset).names == (:col2, :col9) - cols = Tables.columns(dataset) - @test all([length(col)==4 for col in cols]) # all columns must be 4 rows long - @test length(cols) == 2 # 2 columns - @test cols.col2 == ["2002-02-01", "2002-02-01", "2002-02-02", "2002-02-02"] - @test all(cols.col9 .== "02/2030") - - # load dataset partitioned by date column - @test Parquet.parse_date("2002-02-01") == Date("2002-02-01") - @test Parquet.parse_datetime("2002-02-01") == DateTime("2002-02-01") - @test_throws Exception Parquet.parse_date("not date") - @test_throws Exception Parquet.parse_datetime("not date") - dataset_path = joinpath(@__DIR__, "datasets", "date_partition") - dataset = read_parquet(dataset_path) - @test Tables.istable(dataset) - @test Tables.columnaccess(dataset) - @test Tables.schema(dataset).names == (:col2, :col9) - cols = Tables.columns(dataset) - @test all([length(col)==4 for col in cols]) # all columns must be 4 rows long - @test length(cols) == 2 # 2 columns - @test cols.col2 == [DateTime("2002-02-01"), DateTime("2002-02-01"), DateTime("2002-02-02"), DateTime("2002-02-02")] - @test all(cols.col9 .== "02/2030") - - # load dataset without metadata file - mktempdir() do path - new_dataset = joinpath(path, "datasets") - cp(dataset_path, new_dataset) - metafile = joinpath(new_dataset, "_common_metadata") - rm(metafile) - dataset = read_parquet(new_dataset) - @test Tables.istable(dataset) - @test Tables.columnaccess(dataset) - end - end -end - -@testset "load files" begin - test_load_all_pages() - test_decode_all_pages() - test_load_boolean_and_ts() - test_load_nested() - test_load_multiple_rowgroups() - test_load_file() - test_load_at_offset() - test_mmap_mode() - test_zero_rows() - test_dataset() -end diff --git a/test/test_writer.jl b/test/test_writer.jl deleted file mode 100644 index e61fa1c..0000000 --- a/test/test_writer.jl +++ /dev/null @@ -1,98 +0,0 @@ -using Parquet -using Test -using Random - -if VERSION < v"1.3" - using Missings: nonmissingtype -end - -Random.seed!(1234567) - -function test_written(tbl, tmpfile) - pf = Parquet.File(tmpfile) - N = length(tbl.int32) - - # the file is very small so only one rowgroup - col_chunks = columns(pf, 1) - - for (colnum, col_chunk) in enumerate(col_chunks) - correct_vals = tbl[colnum] - coltype = eltype(correct_vals) - jtype = Parquet.elemtype(Parquet.elem(schema(pf), colname(pf,col_chunk))) - ccpv = Parquet.ColumnChunkPageValues(pf, col_chunk, jtype) - resultdata,nextpos = iterate(ccpv) - - if Missing <: coltype - @test ismissing.(correct_vals) == (resultdata.defn_level.data .== 0) - end - - non_missing_vals = collect(skipmissing(correct_vals)) - - if nonmissingtype(coltype) == String - @test all(non_missing_vals .== String.(resultdata.value.data)) - else - @test all(non_missing_vals .== resultdata.value.data) - end - end - - # test with BatchedColumnsCursor - cc = Parquet.BatchedColumnsCursor(pf) - vals, _ = iterate(cc) - @test length(vals) == 12 - @test length(vals.boolm) == N - for idx in 1:N - @test ismissing(vals.boolm[idx]) == ismissing(tbl.boolm[idx]) - if !ismissing(vals.boolm[idx]) - @test vals.boolm[idx] == tbl.boolm[idx] - end - end - - # clean up - close(pf) -end - -function make_table(N::Int=1000) - tbl = ( - int32 = rand(Int32, N), - int64 = rand(Int64, N), - float32 = rand(Float32, N), - float64 = rand(Float64, N), - bool = rand(Bool, N), - string = [randstring(8) for i in 1:N], - int32m = rand([missing, rand(Int32, 10)...], N), - int64m = rand([missing, rand(Int64, 10)...], N), - float32m = rand([missing, rand(Float32, 10)...], N), - float64m = rand([missing, rand(Float64, 10)...], N), - boolm = rand([missing, true, false], N), - stringm = rand([missing, "abc", "def", "ghi"], N) - ) -end - -function test_write() - tbl = make_table() - tmpfile = tempname()*".parquet" - write_parquet(tmpfile, tbl) - test_written(tbl, tmpfile) -end - -function test_write_via_buffer() - tbl = make_table() - tmpfile = tempname()*".parquet" - - # write to io buffer first - io = IOBuffer() - write_parquet(io, tbl) - - # then dump io buffer to disk - open(tmpfile, "w") do f - write(f, take!(io)) - end - - #and now test round-trip via file - test_written(tbl, tmpfile) -end - -@testset "writer" begin - test_write() - test_write_via_buffer() -end diff --git a/test/thrift.jl b/test/thrift.jl new file mode 100644 index 0000000..e6eb3ac --- /dev/null +++ b/test/thrift.jl @@ -0,0 +1,375 @@ +if !@isdefined(TH) + const TH = Parquet.Thrift +end + +function thriftbytes(f) + w = TH.Writer() + f(w) + return w.buffer +end + +struct ThriftShiftedBytes <: AbstractVector{UInt8} + bytes::Vector{UInt8} + offset::Int +end + +Base.size(bytes::ThriftShiftedBytes) = (length(bytes.bytes),) +Base.axes(bytes::ThriftShiftedBytes) = + (bytes.offset:(bytes.offset + length(bytes.bytes) - 1),) +Base.IndexStyle(::Type{ThriftShiftedBytes}) = IndexLinear() + +function Base.getindex(bytes::ThriftShiftedBytes, index::Int) + checkbounds(bytes, index) + return bytes.bytes[index - bytes.offset + 1] +end + +struct ThriftShiftedValue + bytes::ThriftShiftedBytes +end + +function TH.encode!(writer::TH.Writer, value::ThriftShiftedValue) + append!(writer.buffer, value.bytes) + return +end + +function thriftallocationmetadata(groups::Int) + MD = Parquet.Metadata + rowgroup = MD.RowGroup(columns=MD.ColumnChunk[], + total_byte_size=Int64(0), num_rows=Int64(0)) + return MD.FileMetaData( + version=Int32(1), + schema=MD.SchemaElement[MD.SchemaElement(name="schema", + num_children=Int32(0))], + num_rows=Int64(0), + row_groups=fill(rowgroup, groups), + created_by="allocation-test", + ) +end + +function thriftwriterallocations(value) + exact = TH._encodedsize(value) + bytes = Vector{UInt8}(undef, Int(exact)) + TH._encodedsize(value) + TH._encodefixed!(bytes, value) + count = @allocated TH._encodedsize(value) + fixed = @allocated TH._encodefixed!(bytes, value) + return count, fixed +end + +@testset "compact protocol scalars" begin + @test thriftbytes(w -> TH.writei32!(w, Int32(0))) == UInt8[0x00] + @test thriftbytes(w -> TH.writei32!(w, Int32(-1))) == UInt8[0x01] + @test thriftbytes(w -> TH.writei32!(w, Int32(1))) == UInt8[0x02] + @test thriftbytes(w -> TH.writei32!(w, Int32(300))) == UInt8[0xd8, 0x04] + @test thriftbytes(w -> TH.writei64!(w, Int64(-2))) == UInt8[0x03] + @test thriftbytes(w -> TH.writei16!(w, Int16(32767))) == UInt8[0xfe, 0xff, 0x03] + @test thriftbytes(w -> TH.writedouble!(w, 1.0)) == UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f] + @test thriftbytes(w -> TH.writei8!(w, Int8(-1))) == UInt8[0xff] + @test thriftbytes(w -> TH.writestring!(w, "ab")) == UInt8[0x02, 0x61, 0x62] + @test thriftbytes(w -> TH.writebool!(w, true)) == UInt8[0x01] + @test thriftbytes(w -> TH.writebool!(w, false)) == UInt8[0x02] + for value in (Int32(0), Int32(-1), Int32(1), typemin(Int32), typemax(Int32), Int32(300), Int32(-300)) + @test TH.readi32(TH.Reader(thriftbytes(w -> TH.writei32!(w, value)))) == value + end + for value in (Int64(0), Int64(-1), typemin(Int64), typemax(Int64), Int64(2)^40, -Int64(2)^40) + @test TH.readi64(TH.Reader(thriftbytes(w -> TH.writei64!(w, value)))) == value + end + for value in (typemin(Int16), Int16(-1), Int16(0), typemax(Int16)) + @test TH.readi16(TH.Reader(thriftbytes(w -> TH.writei16!(w, value)))) == value + end + for value in (typemin(Int8), Int8(0), typemax(Int8)) + @test TH.readi8(TH.Reader(thriftbytes(w -> TH.writei8!(w, value)))) == value + end + for value in (0.0, -0.0, 1.5, -Inf, NaN, floatmax(Float64)) + @test isequal(TH.readdouble(TH.Reader(thriftbytes(w -> TH.writedouble!(w, value)))), value) + end + @test TH.readbool(TH.Reader(UInt8[0x01])) === true + @test TH.readbool(TH.Reader(UInt8[0x02])) === false + @test TH.readstring(TH.Reader(thriftbytes(w -> TH.writestring!(w, "héllo")))) == "héllo" + @test TH.readbinary(TH.Reader(thriftbytes(w -> TH.writebinary!(w, UInt8[0xff, 0x00])))) == UInt8[0xff, 0x00] + @test TH.readstring(TH.Reader(UInt8[0x00])) == "" + @test TH.readi32(TH.Reader(UInt8[0xff, 0xff, 0xff, 0xff, 0x0f])) == typemin(Int32) + @test TH.readi64(TH.Reader(vcat(fill(0xff, 9), UInt8[0x01]))) == typemin(Int64) + r = TH.Reader(UInt8[0x02, 0x61, 0x62, 0x07]) + @test TH.readstring(r) == "ab" + @test TH.consumed(r) == 3 + @test TH.remaining(r) == 1 +end + +@testset "exact Compact Thrift writer buffers" begin + dynamic = UInt8[0xaa] + writer = TH.Writer(dynamic) + TH.writei32!(writer, Int32(300)) + @test writer.buffer === dynamic + @test dynamic == UInt8[0xaa, 0xd8, 0x04] + + value = ThriftShiftedValue(ThriftShiftedBytes(UInt8[0x10, 0x20, 0x30], 7)) + exact = TH._encodedsize(value) + fixed = Vector{UInt8}(undef, exact) + @test exact == 3 + @test TH._encodefixed!(fixed, value) === fixed + @test fixed == UInt8[0x10, 0x20, 0x30] + + counter = TH._CountingBuffer(typemax(Int64)) + @test_throws TH._WriteCountOverflow push!(counter, UInt8(0)) + @test counter.count == typemax(Int64) + short = Vector{UInt8}(undef, 2) + @test_throws AssertionError TH._encodefixed!(short, value) + long = Vector{UInt8}(undef, 4) + @test_throws AssertionError TH._encodefixed!(long, value) +end + +@testset "exact Compact Thrift writer allocations" begin + small = thriftallocationmetadata(0) + large = thriftallocationmetadata(256) + thriftwriterallocations(small) + thriftwriterallocations(large) + smallcount, smallfixed = thriftwriterallocations(small) + largecount, largefixed = thriftwriterallocations(large) + threshold = 1024 + @test smallcount <= threshold + @test largecount <= threshold + @test largecount <= smallcount + 512 + @test smallfixed <= threshold + @test largefixed <= threshold + @test largefixed <= smallfixed + 512 +end + +@testset "field headers" begin + w = TH.Writer() + lastid = TH.writefieldheader!(w, Int16(0), Int16(1), TH.I32) + lastid = TH.writefieldheader!(w, lastid, Int16(16), TH.BINARY) + lastid = TH.writefieldheader!(w, lastid, Int16(32), TH.STRUCT) + lastid = TH.writefieldheader!(w, lastid, Int16(3), TH.BOOL_TRUE) + lastid = TH.writefieldheader!(w, lastid, Int16(32767), TH.BINARY) + TH.writestop!(w) + @test w.buffer == UInt8[0x15, 0xf8, 0x0c, 0x40, 0x01, 0x06, 0x08, 0xfe, 0xff, 0x03, 0x00] + r = TH.Reader(w.buffer) + @test TH.readfieldheader(r, Int16(0)) == (Int16(1), TH.I32) + @test TH.readfieldheader(r, Int16(1)) == (Int16(16), TH.BINARY) + @test TH.readfieldheader(r, Int16(16)) == (Int16(32), TH.STRUCT) + @test TH.readfieldheader(r, Int16(32)) == (Int16(3), TH.BOOL_TRUE) + @test TH.readfieldheader(r, Int16(3)) == (Int16(32767), TH.BINARY) + @test TH.readfieldheader(r, Int16(32767)) == (Int16(0), TH.STOP) + @test TH.remaining(r) == 0 + @test r.headerpos == 11 && r.previd == 32767 + @test_throws Parquet.FormatError TH.readfieldheader(TH.Reader(UInt8[0x1d]), Int16(0)) + @test_throws Parquet.FormatError TH.readfieldheader(TH.Reader(UInt8[0x1e]), Int16(0)) + @test_throws Parquet.FormatError TH.readfieldheader(TH.Reader(UInt8[0x1f]), Int16(0)) + @test_throws Parquet.FormatError TH.readfieldheader(TH.Reader(UInt8[0x10]), Int16(0)) + @test_throws Parquet.FormatError TH.readfieldheader(TH.Reader(UInt8[0xf5]), Int16(32760)) + @test_throws Parquet.FormatError TH.readfieldheader(TH.Reader(UInt8[0x05]), Int16(0)) + @test_throws Parquet.FormatError TH.readfieldheader(TH.Reader(UInt8[]), Int16(0)) +end + +@testset "containers" begin + @test thriftbytes(w -> TH.writelist!(w, Int32[1, 2])) == UInt8[0x25, 0x02, 0x04] + @test thriftbytes(w -> TH.writelist!(w, Bool[true, false])) == UInt8[0x21, 0x01, 0x02] + long = Int32.(1:20) + bytes = thriftbytes(w -> TH.writelist!(w, long)) + @test bytes[1:2] == UInt8[0xf5, 0x14] + @test TH.readlist(TH.Reader(bytes), Int32) == long + @test TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, Bool[true, false]))), Bool) == [true, false] + @test TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, ["a", "bc"]))), String) == ["a", "bc"] + @test TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, Int8[-1, 1]))), Int8) == Int8[-1, 1] + @test TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, Int16[-1, 1]))), Int16) == Int16[-1, 1] + @test TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, [UInt8[1], UInt8[]]))), Vector{UInt8}) == [UInt8[1], UInt8[]] + nested = [Int64[1], Int64[], Int64[-5, 7]] + @test TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, nested))), Vector{Int64}) == nested + doubles = [1.5, NaN] + @test isequal(TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, doubles))), Float64), doubles) + r = TH.Reader(UInt8[0x25, 0x02, 0x04]) + @test TH.readlist(r, String) === nothing + @test TH.consumed(r) == 0 + @test TH.readlist(r, Int32) == Int32[1, 2] + @test r.depth == 0 + pairs = ["a" => Int64(1), "b" => Int64(-1)] + bytes = thriftbytes(w -> TH.writemap!(w, pairs)) + @test bytes == UInt8[0x02, 0x86, 0x01, 0x61, 0x02, 0x01, 0x62, 0x01] + @test TH.readmap(TH.Reader(bytes), String, Int64) == pairs + @test thriftbytes(w -> TH.writemap!(w, Pair{String,Int64}[])) == UInt8[0x00] + @test TH.readmap(TH.Reader(UInt8[0x00]), String, Int64) == Pair{String,Int64}[] + r = TH.Reader(bytes) + @test TH.readmap(r, Int32, Int64) === nothing + @test TH.consumed(r) == 0 + listofmaps = [[Int32(1) => "x"], Pair{Int32,String}[]] + @test TH.readlist(TH.Reader(thriftbytes(w -> TH.writelist!(w, listofmaps))), Vector{Pair{Int32,String}}) == listofmaps + mapofbools = [true => [false]] + @test TH.readmap(TH.Reader(thriftbytes(w -> TH.writemap!(w, mapofbools))), Bool, Vector{Bool}) == mapofbools +end + +@testset "skip and raw field capture" begin + bytes = thriftbytes() do w + lastid = TH.writefieldheader!(w, Int16(0), Int16(1), TH.BOOL_TRUE) + lastid = TH.writefieldheader!(w, lastid, Int16(2), TH.BYTE) + TH.writei8!(w, Int8(7)) + lastid = TH.writefieldheader!(w, lastid, Int16(3), TH.I16) + TH.writei16!(w, Int16(-300)) + lastid = TH.writefieldheader!(w, lastid, Int16(4), TH.I32) + TH.writei32!(w, Int32(123456)) + lastid = TH.writefieldheader!(w, lastid, Int16(5), TH.I64) + TH.writei64!(w, Int64(-1)) + lastid = TH.writefieldheader!(w, lastid, Int16(6), TH.DOUBLE) + TH.writedouble!(w, 2.5) + lastid = TH.writefieldheader!(w, lastid, Int16(7), TH.BINARY) + TH.writebinary!(w, UInt8[1, 2, 3]) + lastid = TH.writefieldheader!(w, lastid, Int16(8), TH.LIST) + TH.writelist!(w, Int32[1, 2, 3]) + lastid = TH.writefieldheader!(w, lastid, Int16(9), TH.SET) + TH.writelist!(w, ["x"]) + lastid = TH.writefieldheader!(w, lastid, Int16(10), TH.MAP) + TH.writemap!(w, [Int32(1) => UInt8[9]]) + lastid = TH.writefieldheader!(w, lastid, Int16(11), TH.STRUCT) + TH.writefieldheader!(w, Int16(0), Int16(1), TH.BINARY) + TH.writestring!(w, "inner") + TH.writestop!(w) + lastid = TH.writefieldheader!(w, lastid, Int16(12), TH.BOOL_FALSE) + TH.writestop!(w) + end + r = TH.Reader(bytes) + TH.skipstruct!(r) + @test TH.remaining(r) == 0 + @test r.depth == 0 + r = TH.Reader(bytes) + TH.enter!(r) + lastid = Int16(0) + fields = TH.RawField[] + while true + id, ty = TH.readfieldheader(r, lastid) + ty == TH.STOP && break + lastid = id + push!(fields, TH.readrawfield(r, id, ty)) + end + @test [f.id for f in fields] == Int16.(1:12) + @test fields[1].bytes == UInt8[0x11] && isempty(TH.payload(fields[1])) + @test fields[7].bytes == UInt8[0x18, 0x03, 0x01, 0x02, 0x03] + @test TH.payload(fields[7]) == UInt8[0x03, 0x01, 0x02, 0x03] + @test fields[12].type == TH.BOOL_FALSE && isempty(TH.payload(fields[12])) + @test all(f -> f.headerlength == 1, fields) + @test [f.previd for f in fields] == Int16.(0:11) + @test fields[3] == fields[3] && hash(fields[3]) == hash(fields[3]) && fields[3] != fields[4] + w = TH.Writer() + lastid = Int16(0) + for field in fields + lastid = TH.writeraw!(w, lastid, field) + end + TH.writestop!(w) + @test w.buffer == bytes + # a raw field re-emitted after a different predecessor gets a synthesized header + w = TH.Writer() + @test TH.writeraw!(w, Int16(0), fields[7]) == 7 + @test w.buffer == UInt8[0x78, 0x03, 0x01, 0x02, 0x03] + # regression: skipping a binary field must consume its length prefix (found with ARROW-GH-41317) + kv = thriftbytes() do w + TH.writefieldheader!(w, Int16(0), Int16(1), TH.BINARY) + TH.writestring!(w, "boolean") + TH.writestop!(w) + end + r = TH.Reader(kv) + TH.skipstruct!(r) + @test TH.remaining(r) == 0 +end + +@testset "malformed input" begin + F = Parquet.FormatError + @test_throws F TH.readbyte(TH.Reader(UInt8[])) + @test_throws F TH.readi32(TH.Reader(UInt8[0x80])) + @test_throws F TH.readi64(TH.Reader(UInt8[0x80, 0x80, 0x80])) + @test_throws F TH.readi32(TH.Reader(UInt8[0x80, 0x80, 0x80, 0x80, 0x80, 0x00])) + @test_throws F TH.readi32(TH.Reader(UInt8[0x80, 0x80, 0x80, 0x80, 0x10])) + @test_throws F TH.readi64(TH.Reader(vcat(fill(0x80, 9), UInt8[0x02]))) + @test_throws F TH.readi64(TH.Reader(vcat(fill(0x80, 10), UInt8[0x00]))) + @test_throws F TH.readi16(TH.Reader(thriftbytes(w -> TH.writei32!(w, Int32(40000))))) + @test_throws F TH.readdouble(TH.Reader(fill(0x00, 7))) + @test_throws F TH.readbool(TH.Reader(UInt8[0x00])) + @test_throws F TH.readbool(TH.Reader(UInt8[0x03])) + @test_throws F TH.readlist(TH.Reader(UInt8[0x21, 0x01, 0x00]), Bool) + @test_throws F TH.skiplist!(TH.Reader(UInt8[0x21, 0x01, 0x00])) + @test_throws F TH.readstring(TH.Reader(UInt8[0x05, 0x61])) + @test_throws F TH.readstring(TH.Reader(UInt8[0xff, 0xff, 0xff, 0xff, 0x0f])) + @test_throws F TH.readlist(TH.Reader(UInt8[0xf5, 0xff, 0xff, 0xff, 0xff, 0x0f]), Int32) + @test_throws F TH.readlist(TH.Reader(UInt8[0x2d, 0x00, 0x00]), Int32) + @test_throws F TH.readlistheader(TH.Reader(UInt8[0x20])) + @test_throws F TH.readmap(TH.Reader(UInt8[0x01, 0xd5, 0x00, 0x00]), Int32, Int32) + @test_throws F TH.readmap(TH.Reader(UInt8[0x01, 0x5d, 0x00, 0x00]), Int32, Int32) + @test_throws F TH.readmap(TH.Reader(UInt8[0x01]), Int32, Int32) + @test_throws F TH.skipvalue!(TH.Reader(UInt8[0x00]), UInt8(13)) + @test_throws F TH.skipvalue!(TH.Reader(UInt8[0x00]), UInt8(0)) + @test_throws F TH.skipstruct!(TH.Reader(UInt8[0x15])) + @test_throws F TH.skipstruct!(TH.Reader(UInt8[0x18, 0x05, 0x61])) + @test_throws F TH.skipstruct!(TH.Reader(UInt8[0x17, 0x00])) + @test_throws F TH.skipstruct!(TH.Reader(UInt8[0x1c, 0x1d, 0x00, 0x00])) +end + +@testset "resource limits" begin + L = Parquet.LimitError + F = Parquet.FormatError + small = Parquet.Limits(max_string_bytes=4) + err = try + TH.readstring(TH.Reader(UInt8[0x05, 0x61, 0x62, 0x63, 0x64, 0x65]; limits=small)) + nothing + catch e + e + end + @test err isa L && err.resource == :string_bytes && err.requested == 5 && err.maximum == 4 + @test TH.readstring(TH.Reader(UInt8[0x04, 0x61, 0x62, 0x63, 0x64]; limits=small)) == "abcd" + @test_throws L TH.skipvalue!(TH.Reader(UInt8[0x05, 0x61, 0x62, 0x63, 0x64, 0x65]; limits=small), TH.BINARY) + @test_throws F TH.readstring(TH.Reader(UInt8[0x80, 0x80, 0x40])) + @test_throws L TH.readstring(TH.Reader(UInt8[0xff, 0xff, 0xff, 0xff, 0x07])) + few = Parquet.Limits(max_container_elements=3) + @test_throws L TH.readlist(TH.Reader(UInt8[0x45, 0x00, 0x00, 0x00, 0x00]; limits=few), Int32) + @test_throws L TH.skiplist!(TH.Reader(UInt8[0x45, 0x00, 0x00, 0x00, 0x00]; limits=few)) + @test TH.readlist(TH.Reader(UInt8[0x35, 0x00, 0x00, 0x00]; limits=few), Int32) == Int32[0, 0, 0] + @test_throws F TH.readlist(TH.Reader(UInt8[0xf5, 0x80, 0x80, 0x80, 0x01]), Int32) + @test_throws L TH.readlist(TH.Reader(UInt8[0xf5, 0xff, 0xff, 0xff, 0xff, 0x07]), Int32) + @test_throws F TH.readlist(TH.Reader(vcat(UInt8[0xf7, 0x80, 0x08], zeros(UInt8, 64))), Float64) + @test_throws L TH.readmap(TH.Reader(UInt8[0x04, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; limits=few), Int32, Int32) + @test_throws F TH.readmap(TH.Reader(UInt8[0x7f, 0x55, 0x00, 0x00]), Int32, Int32) + shallow = Parquet.Limits(max_metadata_depth=2) + @test TH.readlist(TH.Reader(UInt8[0x19, 0x15, 0x00]; limits=shallow), Vector{Int32}) == [Int32[0]] + @test_throws L TH.readlist(TH.Reader(UInt8[0x19, 0x19, 0x15, 0x00]; limits=shallow), Vector{Vector{Int32}}) + err = try + TH.skiplist!(TH.Reader(vcat(fill(0x19, 199), UInt8[0x05]))) + nothing + catch e + e + end + @test err isa L && err.resource == :metadata_depth && err.requested == 129 && err.maximum == 128 + @test_throws L TH.skipstruct!(TH.Reader(fill(0x1c, 200))) + @test_throws L TH.skipstruct!(TH.Reader(vcat(UInt8[0x1b, 0x01, 0xcc], fill(0x1c, 200)))) + r = TH.Reader(vcat(fill(0x19, 100), UInt8[0x05])) + TH.skiplist!(r) + @test r.depth == 0 && TH.remaining(r) == 0 + + binarybudget = Parquet._LiveByteBudget( + Parquet.Limits(max_materialized_bytes=100)) + binaryreader = TH.Reader(vcat(UInt8[0x64], fill(UInt8(0x61), 100)); + budget=binarybudget) + @test_throws L TH.readbinary(binaryreader) + @test Parquet._budgetused(binarybudget) == 0 + + listbudget = Parquet._LiveByteBudget( + Parquet.Limits(max_materialized_bytes=70)) + listreader = TH.Reader(UInt8[0x25, 0x00, 0x00]; budget=listbudget) + @test_throws L TH.readlist(listreader, Int32) + @test Parquet._budgetused(listbudget) == 0 + + decodebudget = Parquet._LiveByteBudget(Parquet.Limits()) + @test_throws F TH.decode(UInt8[0x00], Parquet.Metadata.FileMetaData; + budget=decodebudget) + @test Parquet._budgetused(decodebudget) == 0 +end + +@testset "reader ranges and byte sources" begin + bytes = UInt8[0x15, 0x02, 0x00] + r = TH.Reader(bytes, 2, 3) + @test TH.readi32(r) == 1 && TH.remaining(r) == 1 && TH.consumed(r) == 1 + @test_throws BoundsError TH.Reader(bytes, 0, 3) + @test_throws BoundsError TH.Reader(bytes, 1, 4) + @test_throws ArgumentError TH.Reader(bytes, 3, 1) + @test TH.remaining(TH.Reader(bytes, 2, 1)) == 0 + slice = Parquet.readrange(Parquet.source(bytes), 0, 3) + @test TH.readfieldheader(TH.Reader(slice), Int16(0)) == (Int16(1), TH.I32) + @test TH.readlist(TH.Reader(view(UInt8[0x00, 0x25, 0x02, 0x04], 2:4)), Int32) == Int32[1, 2] + @test TH.readstring(TH.Reader(view(UInt8[0x00, 0x02, 0x61, 0x62], 2:4))) == "ab" +end diff --git a/test/write.jl b/test/write.jl new file mode 100644 index 0000000..387f5f6 --- /dev/null +++ b/test/write.jl @@ -0,0 +1,1196 @@ +using Dates +using SHA + +struct DeferredWriterName <: AbstractString + converted::Base.RefValue{Bool} + bytes::Int +end + +function Base.ncodeunits(name::DeferredWriterName) + return name.bytes +end + +function Base.String(name::DeferredWriterName) + name.converted[] = true + return "deferred" +end + +function writtenpages(bytes::Vector{UInt8}, column::Int) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + chunk = metadata.row_groups[1].columns[column].meta_data + start, stop = Parquet._chunkrange(chunk, file.footer.offset) + pages = NamedTuple[] + position = start + while position < stop + frame = Parquet.readpage(file.source, position, stop, Parquet.Limits()) + push!(pages, (header=frame.header, payload=collect(frame.payload))) + position = Parquet.pageend(frame) + end + close(file) + return pages, chunk +end + +struct FooterTestValue{B<:AbstractVector{UInt8}} + bytes::B +end + +function TH.encode!(writer::TH.Writer, value::FooterTestValue) + append!(writer.buffer, value.bytes) + return +end + +mutable struct FooterPhaseValue + actions::Vector{Any} + calls::Int +end + +function TH.encode!(writer::TH.Writer, value::FooterPhaseValue) + value.calls += 1 + action = value.actions[value.calls] + action isa Exception && throw(action) + append!(writer.buffer, action) + return +end + +struct FooterVirtualBytes <: AbstractVector{UInt8} + count::Int +end + +Base.size(bytes::FooterVirtualBytes) = (bytes.count,) +Base.getindex(::FooterVirtualBytes, ::Int) = + throw(AssertionError("virtual footer bytes must not be materialized")) + +struct FooterOverflowValue end + +function TH.encode!(writer::TH.Writer, ::FooterOverflowValue) + append!(writer.buffer, FooterVirtualBytes(typemax(Int))) + push!(writer.buffer, UInt8(0)) + return +end + +function minimalfootermetadata(; unknown_fields=()) + return MD.FileMetaData( + version=Int32(1), + schema=MD.SchemaElement[MD.SchemaElement(name="schema", + num_children=Int32(0))], + num_rows=Int64(0), + row_groups=MD.RowGroup[], + created_by="footer-test", + unknown_fields=unknown_fields, + ) +end + +function boundedfooter(value; limits=Parquet.Limits()) + budget = Parquet._LiveByteBudget(limits) + bytes, charge = Parquet._writeencodefooter(value, limits, budget) + @test Parquet._budgetused(budget) == charge + Parquet._release!(budget, charge) + @test Parquet._budgetused(budget) == 0 + return bytes +end + +@testset "exact bounded footer encoding" begin + metadata = minimalfootermetadata() + dynamic = TH.encode(metadata) + exact = Int64(length(dynamic)) + valuecharge = Parquet._materializedarraybytes(UInt8, exact) + controlcharge = Parquet._WRITE_FOOTER_CONTROL_BYTES + entry = Int64(17) + maximum = entry + controlcharge + valuecharge + limits = Parquet.Limits(max_footer_bytes=exact, + max_materialized_bytes=maximum) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, entry) + bytes, charge = Parquet._writeencodefooter(metadata, limits, budget) + @test bytes == dynamic + @test charge == valuecharge + @test Parquet._budgetused(budget) == entry + charge + Parquet._release!(budget, charge) + @test Parquet._budgetused(budget) == entry + + footerbudget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(footerbudget, entry) + footererr = try + Parquet._writeencodefooter(metadata, + Parquet.Limits(max_footer_bytes=exact - 1), footerbudget) + nothing + catch caught + caught + end + @test footererr isa Parquet.LimitError + @test footererr.resource == :footer_bytes + @test footererr.requested == exact + @test footererr.maximum == exact - 1 + @test Parquet._budgetused(footerbudget) == entry + + tightlimits = Parquet.Limits(max_footer_bytes=exact, + max_materialized_bytes=maximum - 1) + tightbudget = Parquet._LiveByteBudget(tightlimits) + Parquet._reserve!(tightbudget, entry) + materialerr = try + Parquet._writeencodefooter(metadata, tightlimits, tightbudget) + nothing + catch caught + caught + end + @test materialerr isa Parquet.LimitError + @test materialerr.resource == :materialized_bytes + @test materialerr.requested == maximum + @test materialerr.maximum == maximum - 1 + @test Parquet._budgetused(tightbudget) == entry + + verbatim = TH.RawField(Int16(10), TH.I32, Int16(6), Int8(2), + UInt8[TH.I32, 0x14, 0x02]) + preserved = minimalfootermetadata(unknown_fields=(verbatim,)) + preservedbytes = boundedfooter(preserved) + @test preservedbytes == TH.encode(preserved) + @test preservedbytes[(end - 3):end] == UInt8[TH.I32, 0x14, 0x02, TH.STOP] + synthesized = TH.RawField(Int16(10), TH.I32, Int16(5), Int8(2), + UInt8[TH.I32, 0x14, 0x02]) + canonical = minimalfootermetadata(unknown_fields=(synthesized,)) + canonicalbytes = boundedfooter(canonical) + @test canonicalbytes == TH.encode(canonical) + @test canonicalbytes[(end - 2):end] == UInt8[0x45, 0x02, TH.STOP] + + nestedraw = TH.RawField(Int16(11), TH.I32, Int16(5), Int8(2), + UInt8[TH.I32, 0x16, 0x0e]) + topraw = TH.RawField(Int16(12), TH.I32, Int16(6), Int8(2), + UInt8[TH.I32, 0x18, 0x12]) + nestedmetadata = MD.FileMetaData( + version=Int32(1), + schema=MD.SchemaElement[MD.SchemaElement(name="schema", + num_children=Int32(0), unknown_fields=(nestedraw,))], + num_rows=Int64(0), + row_groups=MD.RowGroup[], + created_by="footer-test", + unknown_fields=(topraw,), + ) + original = TH.encode(nestedmetadata) + decoded = TH.decode(original, MD.FileMetaData) + @test only(decoded.schema).unknown_fields == [nestedraw] + @test decoded.unknown_fields == [topraw] + @test TH.encode(decoded) == original + @test boundedfooter(decoded) == original + + rowgroup = MD.RowGroup(columns=MD.ColumnChunk[], total_byte_size=Int64(0), + num_rows=Int64(0)) + large = MD.FileMetaData(version=Int32(1), schema=metadata.schema, + num_rows=Int64(0), row_groups=fill(rowgroup, 4096), + created_by=metadata.created_by) + largedynamic = TH.encode(large) + largeexact = Int64(length(largedynamic)) + largebytes = boundedfooter(large; limits=Parquet.Limits( + max_footer_bytes=largeexact, + max_materialized_bytes=Parquet._WRITE_FOOTER_CONTROL_BYTES + + Parquet._materializedarraybytes(UInt8, largeexact))) + @test largebytes == largedynamic + + for (actions, expected) in ( + (Any[UInt8[1], UInt8[1, 2]], AssertionError), + (Any[UInt8[1, 2], UInt8[1]], AssertionError)) + phase = FooterPhaseValue(actions, 0) + phasebudget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(phasebudget, entry) + @test_throws expected Parquet._writeencodefooter(phase, + Parquet.Limits(), phasebudget) + @test Parquet._budgetused(phasebudget) == entry + end + sentinel = ErrorException("footer second-pass sentinel") + throwing = FooterPhaseValue(Any[UInt8[1], sentinel], 0) + throwbudget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(throwbudget, entry) + thrown = try + Parquet._writeencodefooter(throwing, Parquet.Limits(), throwbudget) + nothing + catch caught + caught + end + @test thrown === sentinel + @test Parquet._budgetused(throwbudget) == entry + + countsentinel = ErrorException("footer count-pass sentinel") + countthrowing = FooterPhaseValue(Any[countsentinel], 0) + countbudget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(countbudget, entry) + countthrown = try + Parquet._writeencodefooter(countthrowing, Parquet.Limits(), countbudget) + nothing + catch caught + caught + end + @test countthrown === countsentinel + @test Parquet._budgetused(countbudget) == entry + + wiremaximum = Int64(typemax(UInt32)) + boundary = FooterTestValue(FooterVirtualBytes(Int(wiremaximum))) + boundarylimits = Parquet.Limits(max_footer_bytes=wiremaximum - 1) + boundarybudget = Parquet._LiveByteBudget(boundarylimits) + Parquet._reserve!(boundarybudget, entry) + boundaryerr = try + Parquet._writeencodefooter(boundary, boundarylimits, boundarybudget) + nothing + catch caught + caught + end + @test boundaryerr isa Parquet.LimitError + @test boundaryerr.resource == :footer_bytes + @test boundaryerr.requested == wiremaximum + @test boundaryerr.maximum == wiremaximum - 1 + @test Parquet._budgetused(boundarybudget) == entry + + wireexact = wiremaximum + 1 + virtual = FooterTestValue(FooterVirtualBytes(Int(wireexact))) + for maximum in (wireexact - 1, typemax(Int64)) + virtualbudget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(virtualbudget, entry) + virtualerr = try + Parquet._writeencodefooter(virtual, + Parquet.Limits(max_footer_bytes=maximum), virtualbudget) + nothing + catch caught + caught + end + @test virtualerr isa ArgumentError + @test virtualerr isa ArgumentError && + virtualerr.msg == "Parquet footer exceeds UInt32 bytes" + @test Parquet._budgetused(virtualbudget) == entry + end + + for limits in (Parquet.Limits(), + Parquet.Limits(max_footer_bytes=typemax(Int64))) + overflowbudget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(overflowbudget, entry) + overflowerr = try + Parquet._writeencodefooter(FooterOverflowValue(), limits, + overflowbudget) + nothing + catch caught + caught + end + @test overflowerr isa ArgumentError + @test overflowerr isa ArgumentError && + overflowerr.msg == "Parquet footer exceeds UInt32 bytes" + @test Parquet._budgetused(overflowbudget) == entry + end +end + +@testset "footer limit leaves public destinations unchanged" begin + table = (value=Int32[1, 2],) + reference = Parquet._encodefile(table) + file = Parquet.File(reference) + exact = Int64(length(file.footer.bytes)) + close(file) + exactio = IOBuffer() + Parquet.write(exactio, table; + limits=Parquet.Limits(max_footer_bytes=exact)) + @test take!(exactio) == reference + limits = Parquet.Limits(max_footer_bytes=exact - 1) + sentinel = UInt8[0xde, 0xad, 0xbe, 0xef] + io = IOBuffer() + Base.write(io, sentinel) + err = try + Parquet.write(io, table; limits=limits) + nothing + catch caught + caught + end + @test err isa Parquet.LimitError + @test err.resource == :footer_bytes + @test err.requested == exact + @test take!(io) == sentinel + mktempdir() do directory + existing = joinpath(directory, "existing.parquet") + Base.write(existing, sentinel) + @test_throws Parquet.LimitError Parquet.write(existing, table; + limits=limits) + @test read(existing) == sentinel + missing = joinpath(directory, "missing.parquet") + @test_throws Parquet.LimitError Parquet.write(missing, table; + limits=limits) + @test !ispath(missing) + end +end + +@testset "writer allocation accounting" begin + duplicatebudget = Parquet._LiveByteBudget(Parquet.Limits()) + @test_throws ArgumentError Parquet._validatewritecolumnnames( + Symbol[:duplicate, :duplicate], duplicatebudget) + @test Parquet._budgetused(duplicatebudget) == 0 + + invalidbudget = Parquet._LiveByteBudget(Parquet.Limits()) + @test_throws ArgumentError Parquet._validatewritecolumnnames( + Any[1], invalidbudget) + @test Parquet._budgetused(invalidbudget) == 0 + + converted = Ref(false) + deferred = DeferredWriterName(converted, 10_000) + preflightbudget = Parquet._LiveByteBudget( + Parquet.Limits(max_materialized_bytes=512)) + @test_throws Parquet.LimitError Parquet._validatewritecolumnnames( + AbstractString[deferred], preflightbudget) + @test !converted[] + @test Parquet._budgetused(preflightbudget) == 0 + + source = Parquet._writecolumn(:value, Int32[1]) + field = Parquet._writefieldplan(source) + elements = MD.SchemaElement[ + MD.SchemaElement(name="schema", num_children=Int32(1)), + field.schema..., + ] + schema = Parquet.Schema(elements) + leafbudget = Parquet._LiveByteBudget(Parquet.Limits()) + leaves, leafcharge = Parquet._writeplanleaves( + Parquet.WriteFieldPlan[field], schema, 1, Parquet.Limits(), + leafbudget) + prefixcharge = Parquet._materializedproduct(3, + Parquet._materializedarraybytes(Int64, 2)) + @test length(leaves) == 1 + @test leafcharge == Parquet._writeleafplanbytes(schema.leaves) + + prefixcharge + @test Parquet._budgetused(leafbudget) == leafcharge + Parquet._release!(leafbudget, leafcharge) + @test Parquet._budgetused(leafbudget) == 0 + + zerocolumn = Parquet._writecolumn(:value, Int32[]) + zerofield = Parquet._writefieldplan(zerocolumn) + zerobudget = Parquet._LiveByteBudget(Parquet.Limits()) + zeroplan = Parquet._writeplan(Parquet.WriteFieldPlan[zerofield], 0, + Parquet.Limits(), zerobudget) + onecolumn = Parquet._writecolumn(:value, Int32[1]) + onefield = Parquet._writefieldplan(onecolumn) + onebudget = Parquet._LiveByteBudget(Parquet.Limits()) + oneplan = Parquet._writeplan(Parquet.WriteFieldPlan[onefield], 1, + Parquet.Limits(), onebudget) + rowgroupcharge = Parquet._materializedarraybytes( + Parquet.WriteRowGroupPlan, 1) - Parquet._materializedarraybytes( + Parquet.WriteRowGroupPlan, 0) + @test isempty(zeroplan.rowgroups) + @test length(oneplan.rowgroups) == 1 + @test Parquet._budgetused(onebudget) - Parquet._budgetused(zerobudget) == + Parquet._writeleafplanbytes(oneplan.schema.leaves) + rowgroupcharge + + prefixcharge +end + +@testset "file writer schema ownership" begin + days = Union{Missing,Vector{Union{Missing,Date}}}[ + missing, + Union{Missing,Date}[Date(2020, 1, 2), missing], + ] + input = (id=Int32[1, 2], days=days) + plan = Parquet._writeplan(input) + @test plan.rows == 2 + @test length(plan.rowgroups) == 1 + @test plan.elements[1].num_children == 2 + @test length(plan.schema.root.children) == 2 + @test [element.name for element in plan.elements] == + ["schema", "id", "days", "list", "element"] + rowgroup = only(plan.rowgroups) + @test rowgroup.rows == plan.rows + @test [leaf.ordinal for leaf in rowgroup.leaves] == Int32[1, 2] + @test [leaf.path for leaf in rowgroup.leaves] == + [["id"], ["days", "list", "element"]] + @test [leaf.path for leaf in rowgroup.leaves] == + [leaf.path for leaf in plan.schema.leaves] + @test all(isempty(leaf.column.schema) for leaf in rowgroup.leaves) + @test all(leaf.column.path == leaf.path for leaf in rowgroup.leaves) + bytes = Parquet._encodefile(input) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + @test metadata.schema == plan.elements + @test metadata.num_rows == plan.rows + @test metadata.row_groups[1].num_rows == rowgroup.rows + @test [chunk.meta_data.path_in_schema for chunk in metadata.row_groups[1].columns] == + [leaf.path for leaf in rowgroup.leaves] + close(file) + + leftsource = Parquet._writecolumn(:left, Int32[1, 2]) + rightsource = Parquet._writecolumn(:right, Float64[1, 2]) + left = Parquet._withoutcolumnschema(leftsource, ["pair", "left"]) + right = Parquet._withoutcolumnschema(rightsource, ["pair", "right"]) + group = MD.SchemaElement(name="pair", num_children=Int32(2), + repetition_type=MD.FieldRepetitionType.REQUIRED) + field = Parquet.WriteFieldPlan( + MD.SchemaElement[group, only(leftsource.schema), only(rightsource.schema)], + Parquet.WriteColumn[left, right]) + grouped = Parquet._writeplan(Parquet.WriteFieldPlan[field], 2, Parquet.Limits()) + @test grouped.elements[1].num_children == 1 + @test length(grouped.schema.root.children) == 1 + @test length(grouped.schema.leaves) == 2 + @test [leaf.ordinal for leaf in only(grouped.rowgroups).leaves] == Int32[1, 2] + @test [leaf.path for leaf in only(grouped.rowgroups).leaves] == + [["pair", "left"], ["pair", "right"]] + @test_throws Parquet.UnsupportedFeatureError Parquet._validatewritechoicecount( + only(grouped.rowgroups).leaves, + Parquet.WriteEncodingChoice[Parquet.WriteEncodingChoice(nothing, false)]) + incomplete = Parquet.WriteFieldPlan(field.schema, Parquet.WriteColumn[left]) + @test_throws ArgumentError Parquet._writeplan( + Parquet.WriteFieldPlan[incomplete], 2, Parquet.Limits()) + + emptyplan = Parquet._writeplan((value=Int32[],)) + @test emptyplan.rows == 0 + @test isempty(emptyplan.rowgroups) + @test emptyplan.elements[1].num_children == 1 + for pageversion in (:v1, :v2) + emptybytes = Parquet._encodefile((value=Int32[],); pageversion=pageversion) + emptyfile = Parquet.File(emptybytes) + emptymetadata = TH.decode(emptyfile.footer.bytes, MD.FileMetaData) + @test emptymetadata.num_rows == 0 + @test isempty(emptymetadata.row_groups) + @test [element.name for element in emptymetadata.schema] == ["schema", "value"] + close(emptyfile) + emptytable = Parquet.Table(emptybytes) + @test emptytable.columns.value == Int32[] + close(emptytable) + end + @test_throws ArgumentError Parquet._encodefile((value=Int32[],); + encoding=(unknown=:plain,)) +end + +@testset "writer output is deterministic with row-group offsets" begin + flat = ( + id=Int32[1, 2, 3], + label=Union{Missing,String}["a", missing, "b"], + ) + @test bytes2hex(sha256(Parquet._encodefile(flat; statistics=false))) == + "bd0f5655e9f9aca2a1aa5f1d2721d9ea8f5ca3c9a2714385cca49edfcb7c0256" + @test bytes2hex(sha256(Parquet._encodefile(flat; pageindex=false, + statistics=false))) == + "2d33126eb969251001cb972c25a054f27274de3e6b3684dec5a7fa0a12d9beb4" + dictionary = ( + id=fill(Int64(7), 64), + flag=Union{Missing,Bool}[isodd(index) ? true : missing for index in 1:64], + ) + @test bytes2hex(sha256(Parquet._encodefile(dictionary; + pageversion=:v2, dictionary=true, statistics=false))) == + "4c65c27c3b2eeb850b74ea6c5e76454a44480bcb774724c88bdad955b10a87e5" + @test bytes2hex(sha256(Parquet._encodefile(dictionary; + pageversion=:v2, dictionary=true, pageindex=false, + statistics=false))) == + "979f70b637a8225075e6822836e1bd4d3041fa105a87155a3d4583ac1a108996" + lists = ( + days=Union{Missing,Vector{Union{Missing,Date}}}[ + missing, + Union{Missing,Date}[], + Union{Missing,Date}[Date(2020, 1, 2), missing], + ], + ) + @test bytes2hex(sha256(Parquet._encodefile(lists; + pageversion=:v2, encoding=:plain, statistics=false))) == + "d7a97e35b231befee376faf76c9ee8fb01ffcae60631b7113bd029ac803168b7" + @test bytes2hex(sha256(Parquet._encodefile(lists; + pageversion=:v2, encoding=:plain, pageindex=false, + statistics=false))) == + "c9ba5e006a12d3873e98b8b7948f6c35d2e090b1d3d09ac9036895d892e48afd" +end + +@testset "PLAIN V1 writer metadata" begin + raw = Vector{UInt8}[UInt8[0x00, 0xff], UInt8[], UInt8[0x41]] + table = ( + i32=Int32[1, -2, 3], + i64=Int64[typemin(Int64), 0, typemax(Int64)], + flag=Bool[true, false, true], + f32=Float32[1.5, -0.0, Inf], + f64=Float64[NaN, 2.5, -Inf], + text=["alpha", "", "κ"], + raw=raw, + optional=Union{Missing,Int32}[1, missing, -3], + ) + bytes = Parquet._encodefile(table) + @test bytes[1:4] == Parquet.PARQUET_MAGIC + @test bytes[(end - 3):end] == Parquet.PARQUET_MAGIC + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(copy(file.footer.bytes), Parquet.Metadata.FileMetaData) + @test metadata.version == 1 + @test metadata.num_rows == 3 + @test metadata.created_by == "Parquet.jl version 1.0.0-DEV" + @test length(metadata.row_groups) == 1 + @test length(metadata.schema) == 9 + @test [element.name for element in metadata.schema[2:end]] == collect(String.(keys(table))) + @test metadata.schema[7].logicalType.STRING !== nothing + @test metadata.schema[7].converted_type == Parquet.Metadata.ConvertedType.UTF8 + @test metadata.schema[end].repetition_type == Parquet.Metadata.FieldRepetitionType.OPTIONAL + group = metadata.row_groups[1] + @test group.num_rows == 3 + @test group.total_byte_size == group.total_compressed_size + @test all(chunk.file_offset == 0 for chunk in group.columns) + @test all(chunk.meta_data.codec == Parquet.Metadata.CompressionCodec.UNCOMPRESSED for chunk in group.columns) + @test all(chunk.meta_data.num_values == 3 for chunk in group.columns) + @test all(chunk.meta_data.dictionary_page_offset === nothing for chunk in group.columns) + close(file) +end + + +@testset "DATE and canonical optional LIST writer" begin + days = Union{Missing,Vector{Union{Missing,Date}}}[ + missing, + Union{Missing,Date}[], + Union{Missing,Date}[missing], + Union{Missing,Date}[Date(1970, 1, 1), missing, Date(1969, 12, 31)], + Union{Missing,Date}[Date(2000, 2, 29)], + ] + expectedrepetition = UInt64[0, 0, 0, 0, 1, 1, 0] + expecteddefinition = UInt64[0, 1, 2, 3, 2, 3, 3] + expectedphysical = Int32[0, -1, 11016] + input = (id=Int32[1, 2, 3, 4, 5], days=days) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + @test [element.name for element in metadata.schema] == + ["schema", "id", "days", "list", "element"] + @test metadata.schema[3].logicalType.LIST !== nothing + @test metadata.schema[3].converted_type == MD.ConvertedType.LIST + @test metadata.schema[4].repetition_type == MD.FieldRepetitionType.REPEATED + @test metadata.schema[5].logicalType.DATE !== nothing + @test metadata.schema[5].converted_type == MD.ConvertedType.DATE + close(file) + + pages, chunk = writtenpages(bytes, 2) + @test chunk.path_in_schema == ["days", "list", "element"] + @test chunk.num_values == 7 + @test length(pages) == 1 + page = only(pages) + payload = page.payload + if pageversion === :v1 + header = page.header.data_page_header + @test header.num_values == 7 + repetition, position = Parquet.decode_hybrid(payload, 7, 1; + length_prefix=true) + definition, position = Parquet.decode_hybrid(payload, 7, 2; + offset=position, length_prefix=true) + physical, position = Parquet.decode_plain(Int32, payload, 3; + offset=position) + @test repetition == expectedrepetition + @test definition == expecteddefinition + @test physical == expectedphysical + @test position == length(payload) + 1 + else + header = page.header.data_page_header_v2 + @test header.num_values == 7 + @test header.num_rows == 5 + @test header.num_nulls == 4 + repetitionlength = Int(header.repetition_levels_byte_length) + definitionlength = Int(header.definition_levels_byte_length) + repetitionbytes = @view payload[1:repetitionlength] + definitionstart = repetitionlength + 1 + definitionstop = repetitionlength + definitionlength + definitionbytes = @view payload[definitionstart:definitionstop] + valuebytes = @view payload[(definitionstop + 1):end] + repetition, repetitionposition = Parquet.decode_hybrid( + repetitionbytes, 7, 1) + definition, definitionposition = Parquet.decode_hybrid( + definitionbytes, 7, 2) + physical, valueposition = Parquet.decode_plain(Int32, valuebytes, 3) + @test repetition == expectedrepetition + @test definition == expecteddefinition + @test physical == expectedphysical + @test repetitionposition == length(repetitionbytes) + 1 + @test definitionposition == length(definitionbytes) + 1 + @test valueposition == length(valuebytes) + 1 + end + table = Parquet.Table(bytes) + @test table.columns.id == input.id + @test isequal(table.columns.days, days) + close(table) + end + + for pageversion in (:v1, :v2), encoding in (:plain, :delta_binary_packed) + bytes = Parquet._encodefile((days=days,); pageversion=pageversion, + encoding=encoding, codec=:snappy) + table = Parquet.Table(bytes) + @test isequal(table.columns.days, days) + close(table) + end + + repeated = fill(Union{Missing,Date}[Date(2000, 2, 29), Date(2000, 2, 29)], 256) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile((days=repeated,); pageversion=pageversion, + dictionary=true, codec=:snappy) + _, chunk = writtenpages(bytes, 1) + @test chunk.dictionary_page_offset !== nothing + @test MD.Encoding.RLE_DICTIONARY in chunk.encodings + table = Parquet.Table(bytes) + @test isequal(table.columns.days, repeated) + close(table) + end + + dates = Union{Missing,Date}[Date(1969, 12, 31), missing, Date(2000, 2, 29)] + bytes = Parquet._encodefile((date=dates,); pageversion=:v2) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + @test metadata.schema[2].type_ == MD.Type.INT32 + @test metadata.schema[2].logicalType.DATE !== nothing + @test metadata.schema[2].converted_type == MD.ConvertedType.DATE + close(file) + table = Parquet.Table(bytes) + @test isequal(table.columns.date, dates) + close(table) + + invalid = Vector{Union{Missing,Date,Int32}}[ + Union{Missing,Date,Int32}[Date(2000, 1, 1), Int32(1)], + ] + @test_throws ArgumentError Parquet._encodefile((days=invalid,)) + @test_throws Parquet.LimitError Parquet._encodefile( + (days=[Date[Date(2000, 1, 1), Date(2000, 1, 2)]],); + limits=Parquet.Limits(max_container_elements=1)) +end + +@testset "PLAIN V1 writer page bytes" begin + table = (required=Int32[10, 20, 30], optional=Union{Missing,Int32}[missing, 2, missing]) + bytes = Parquet._encodefile(table) + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(copy(file.footer.bytes), Parquet.Metadata.FileMetaData) + group = metadata.row_groups[1] + required = group.columns[1].meta_data + start = Int(required.data_page_offset) + 1 + reader = Parquet.Thrift.Reader(bytes, start, length(bytes)) + header = Parquet.Thrift.decode(reader, Parquet.Metadata.PageHeader) + @test header.type_ == Parquet.Metadata.PageType.DATA_PAGE + @test header.data_page_header.encoding == Parquet.Metadata.Encoding.PLAIN + payloadstart = start + Parquet.Thrift.consumed(reader) + payload = @view bytes[payloadstart:(payloadstart + header.compressed_page_size - 1)] + Parquet.verifypagechecksum(header.crc, payload) + @test Parquet.decode_plain(Int32, payload, 3) == (Int32[10, 20, 30], length(payload) + 1) + + optional = group.columns[2].meta_data + start = Int(optional.data_page_offset) + 1 + reader = Parquet.Thrift.Reader(bytes, start, length(bytes)) + header = Parquet.Thrift.decode(reader, Parquet.Metadata.PageHeader) + payloadstart = start + Parquet.Thrift.consumed(reader) + payload = @view bytes[payloadstart:(payloadstart + header.compressed_page_size - 1)] + levels, position = Parquet.decode_hybrid(payload, 3, 1; length_prefix=true) + values, position = Parquet.decode_plain(Int32, payload, 1; offset=position) + @test levels == UInt64[0, 1, 0] + @test values == Int32[2] + @test position == length(payload) + 1 + close(file) +end + +@testset "PLAIN V1 writer validation" begin + @test_throws ArgumentError Parquet._encodefile(NamedTuple()) + @test_throws ArgumentError Parquet._encodefile((a=Int32[1], b=Int32[1, 2])) + @test_throws ArgumentError Parquet._encodefile((a=Int128[1, 2],)) + @test_throws Parquet.LimitError Parquet._encodefile((a=Int32[1, 2],); limits=Parquet.Limits(max_container_elements=1)) + splitbytes = Parquet._encodefile((a=fill("large", 10),); + limits=Parquet.Limits(max_page_bytes=10)) + splitpages, _ = writtenpages(splitbytes, 1) + @test length(splitpages) == 10 + splittable = Parquet.Table(splitbytes) + @test splittable.columns.a == fill("large", 10) + close(splittable) + @test_throws Parquet.LimitError Parquet._encodefile((a=["large"],); + limits=Parquet.Limits(max_page_bytes=8)) + @test_throws Parquet.LimitError Parquet._encodefile((a=["large"],); limits=Parquet.Limits(max_string_bytes=4)) + bytes = Parquet._encodefile((a=Int32[1, 2],); checksum=false) + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(copy(file.footer.bytes), Parquet.Metadata.FileMetaData) + start = Int(metadata.row_groups[1].columns[1].meta_data.data_page_offset) + 1 + reader = Parquet.Thrift.Reader(bytes, start, length(bytes)) + header = Parquet.Thrift.decode(reader, Parquet.Metadata.PageHeader) + @test header.crc === nothing + close(file) +end + +@testset "compressed V1 writer" begin + input = ( + id=repeat(Int32[1, 2, 3, 2], 64), + name=repeat(["alpha", "beta", "alpha", "gamma"], 64), + optional=Union{Missing,Float64}[index % 3 == 0 ? missing : index / 10 for index in 1:256], + ) + codecs = ( + (:uncompressed, MD.CompressionCodec.UNCOMPRESSED), + (:snappy, MD.CompressionCodec.SNAPPY), + (:gzip, MD.CompressionCodec.GZIP), + (:brotli, MD.CompressionCodec.BROTLI), + (:zstd, MD.CompressionCodec.ZSTD), + (:lz4_raw, MD.CompressionCodec.LZ4_RAW), + ) + for (name, codec) in codecs + bytes = Parquet._encodefile(input; codec=name, dictionary=true) + @test bytes == Parquet._encodefile(input; codec=name, dictionary=true) + table = Parquet.Table(bytes) + @test table.columns.id == input.id + @test table.columns.name == input.name + @test isequal(table.columns.optional, input.optional) + close(table) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + group = metadata.row_groups[1] + @test all(chunk.meta_data.codec == codec for chunk in group.columns) + @test group.total_byte_size == sum(chunk.meta_data.total_uncompressed_size for chunk in group.columns) + @test group.total_compressed_size == sum(chunk.meta_data.total_compressed_size for chunk in group.columns) + @test group.total_compressed_size == sum(group.columns) do chunk + start, stop = Parquet._chunkrange(chunk.meta_data, file.footer.offset) + return stop - start + end + close(file) + end + @test Parquet._encodefile(input; codec=:SNAPPY) == Parquet._encodefile(input; codec="snappy") + @test Parquet.Table(Parquet._encodefile(input; codec=:gzip, compressionlevel=9)).columns.id == input.id + @test_throws ArgumentError Parquet._encodefile(input; codec=:unknown) + @test_throws ArgumentError Parquet._encodefile(input; codec=:lz4) + @test_throws ArgumentError Parquet._encodefile(input; codec=:lzo) + @test_throws ArgumentError Parquet._encodefile(input; codec=1) + @test_throws ArgumentError Parquet._encodefile(input; compressionlevel=1) + @test_throws ArgumentError Parquet._encodefile(input; codec=:snappy, compressionlevel=1) + @test_throws ArgumentError Parquet._encodefile(input; codec=:gzip, compressionlevel=10) + @test_throws Parquet.LimitError Parquet._encodefile((value=Int32[1],); codec=:snappy, + limits=Parquet.Limits(max_page_bytes=4)) +end + +function publicwritebytes(table; kwargs...) + io = IOBuffer() + Parquet.write(io, table; kwargs...) + return take!(io) +end + +@testset "PLAIN V2 writer" begin + input = ( + required=fill(Int32(7), 256), + optional=Union{Missing,Int32}[isodd(index) ? 7 : missing for index in 1:256], + ) + codecs = ( + (:uncompressed, MD.CompressionCodec.UNCOMPRESSED), + (:snappy, MD.CompressionCodec.SNAPPY), + (:gzip, MD.CompressionCodec.GZIP), + (:brotli, MD.CompressionCodec.BROTLI), + (:zstd, MD.CompressionCodec.ZSTD), + (:lz4_raw, MD.CompressionCodec.LZ4_RAW), + ) + for (name, codec) in codecs + bytes = Parquet._encodefile(input; codec=name, pageversion=:v2) + @test bytes == Parquet._encodefile(input; codec=name, pageversion=:v2) + table = Parquet.Table(bytes) + @test table.columns.required == input.required + @test isequal(table.columns.optional, input.optional) + close(table) + for column in 1:2 + pages, metadata = writtenpages(bytes, column) + @test length(pages) == 1 + header = pages[1].header + data = header.data_page_header_v2 + @test header.type_ == MD.PageType.DATA_PAGE_V2 + @test header.data_page_header === nothing + @test data.num_values == 256 + @test data.num_nulls == (column == 1 ? 0 : 128) + @test data.num_rows == 256 + @test data.encoding == MD.Encoding.PLAIN + @test data.repetition_levels_byte_length == 0 + @test data.definition_levels_byte_length == (column == 1 ? 0 : 33) + @test data.is_compressed == (codec != MD.CompressionCodec.UNCOMPRESSED) + @test header.compressed_page_size == length(pages[1].payload) + @test metadata.codec == codec + @test metadata.encoding_stats == [MD.PageEncodingStats( + page_type=MD.PageType.DATA_PAGE_V2, encoding=MD.Encoding.PLAIN, + count=Int32(1))] + end + end + + small = Parquet._encodefile((value=Int32[1],); codec=:snappy, pageversion=:v2) + smallpages, smallmetadata = writtenpages(small, 1) + smallheader = only(smallpages).header + @test smallmetadata.codec == MD.CompressionCodec.SNAPPY + @test smallheader.data_page_header_v2.is_compressed == false + @test only(smallpages).payload == Parquet.encode_plain(Int32[1]) + + allmissing = Union{Missing,Int32}[missing for _ in 1:8] + missingbytes = Parquet._encodefile((value=allmissing,); codec=:snappy, pageversion=:v2) + missingpages, _ = writtenpages(missingbytes, 1) + missingheader = only(missingpages).header + @test missingheader.data_page_header_v2.is_compressed == false + @test missingheader.compressed_page_size == missingheader.uncompressed_page_size == 2 + missingtable = Parquet.Table(missingbytes) + @test all(ismissing, missingtable.columns.value) + close(missingtable) + + emptybytes = Parquet._encodefile((value=Int32[],); codec=:snappy, pageversion=:v2) + emptyfile = Parquet.File(emptybytes) + emptymetadata = TH.decode(emptyfile.footer.bytes, MD.FileMetaData) + @test emptymetadata.num_rows == 0 + @test isempty(emptymetadata.row_groups) + close(emptyfile) + emptytable = Parquet.Table(emptybytes) + @test isempty(emptytable.columns.value) + close(emptytable) + + io = IOBuffer() + Parquet.write(io, input; codec=:gzip, pageversion="V2") + @test Parquet.Table(take!(io)).columns.required == input.required + @test_throws ArgumentError Parquet._encodefile(input; pageversion=:v3) + @test_throws ArgumentError Parquet._encodefile(input; pageversion=2) +end + +function expectedtablevalues(values::AbstractVector{<:NTuple{N,UInt8}}) where {N} + return Vector{UInt8}[collect(value) for value in values] +end + +function expectedtablevalues( + values::AbstractVector{Union{Missing,NTuple{N,UInt8}}}) where {N} + return Union{Missing,Vector{UInt8}}[ + ismissing(value) ? missing : collect(value) for value in values] +end + +function expectedtablevalues(values::AbstractVector) + return values +end + +function checkencodedfile(input::NamedTuple, encoding::MD.Encoding.T, pageversion::Symbol) + bytes = Parquet._encodefile(input; encoding=encoding, pageversion=pageversion) + table = Parquet.Table(bytes) + for name in keys(input) + expected = expectedtablevalues(getproperty(input, name)) + @test isequal(getproperty(table.columns, name), expected) + end + close(table) + if isempty(first(values(input))) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + @test metadata.num_rows == 0 + @test isempty(metadata.row_groups) + close(file) + return + end + pagetype = pageversion === :v1 ? MD.PageType.DATA_PAGE : MD.PageType.DATA_PAGE_V2 + for (index, name) in enumerate(keys(input)) + pages, metadata = writtenpages(bytes, index) + @test length(pages) == 1 + page = only(pages).header + @test page.type_ == pagetype + pageencoding = pageversion === :v1 ? page.data_page_header.encoding : + page.data_page_header_v2.encoding + @test pageencoding == encoding + values = getproperty(input, name) + optional = Missing <: eltype(values) + expectedencodings = optional && encoding != MD.Encoding.RLE ? + MD.Encoding.T[MD.Encoding.RLE, encoding] : MD.Encoding.T[encoding] + @test metadata.encodings == expectedencodings + @test metadata.dictionary_page_offset === nothing + @test metadata.encoding_stats == MD.PageEncodingStats[ + MD.PageEncodingStats(page_type=pagetype, encoding=encoding, count=Int32(1)), + ] + if pageversion === :v2 + @test page.data_page_header_v2.num_values == length(values) + @test page.data_page_header_v2.num_nulls == count(ismissing, values) + end + end + return +end + +function encodedvaluepayload(values::AbstractVector, encoding::MD.Encoding.T) + column = Parquet._writecolumn(:value, values) + return Parquet._encodedpayload(column, encoding, Parquet.Limits()) +end + +@testset "explicit writer value encodings" begin + plain = ( + required=Int32[1, -2, 3, 0, 9, -10], + optional=Union{Missing,Int32}[1, missing, -2, 0, missing, 4], + ) + delta_binary = ( + i32=Int32[-20, -10, -9, 0, 100, 101], + optional_i32=Union{Missing,Int32}[-20, missing, -9, 0, missing, 101], + i64=Int64[-9_000_000_000, -8_000_000_000, 0, 1, 8_000_000_000, 8_000_000_001], + optional_i64=Union{Missing,Int64}[missing, -8_000_000_000, 0, missing, + 8_000_000_000, 8_000_000_001], + ) + delta_length = ( + text=["alpha", "", "κόσμος", "prefix-a", "prefix-b", "omega"], + optional_text=Union{Missing,String}["alpha", missing, "", "prefix-a", missing, "omega"], + raw=Vector{UInt8}[UInt8[0x00, 0xff], UInt8[], UInt8[0x41], UInt8[1, 2], + UInt8[1, 3], UInt8[9]], + optional_raw=Union{Missing,Vector{UInt8}}[UInt8[0x00, 0xff], missing, UInt8[], + UInt8[1, 2], missing, UInt8[9]], + ) + delta_byte = ( + text=["prefix-a", "prefix-ab", "prefix-b", "", "omega", "omega-2"], + optional_text=Union{Missing,String}["prefix-a", missing, "prefix-b", "", missing, + "omega-2"], + raw=Vector{UInt8}[UInt8[1, 2, 3], UInt8[1, 2, 4], UInt8[1, 5], UInt8[], + UInt8[9], UInt8[9, 2]], + optional_raw=Union{Missing,Vector{UInt8}}[UInt8[1, 2, 3], missing, UInt8[1, 5], + UInt8[], missing, UInt8[9, 2]], + ) + byte_stream_split = ( + i32=Int32[typemin(Int32), -1, 0, 1, 2, typemax(Int32)], + optional_i32=Union{Missing,Int32}[typemin(Int32), missing, 0, 1, missing, typemax(Int32)], + i64=Int64[typemin(Int64), -1, 0, 1, 2, typemax(Int64)], + optional_i64=Union{Missing,Int64}[typemin(Int64), missing, 0, 1, missing, typemax(Int64)], + f32=Float32[-Inf, -0.0, 0.0, 1.5, 2.25, Inf], + optional_f32=Union{Missing,Float32}[-Inf, missing, 0.0, 1.5, missing, Inf], + f64=Float64[-Inf, -0.0, 0.0, 1.5, 2.25, Inf], + optional_f64=Union{Missing,Float64}[-Inf, missing, 0.0, 1.5, missing, Inf], + ) + rle = ( + required=Bool[true, true, false, false, true, false], + optional=Union{Missing,Bool}[true, missing, false, false, missing, true], + ) + cases = ( + (plain, MD.Encoding.PLAIN), + (delta_binary, MD.Encoding.DELTA_BINARY_PACKED), + (delta_length, MD.Encoding.DELTA_LENGTH_BYTE_ARRAY), + (delta_byte, MD.Encoding.DELTA_BYTE_ARRAY), + (byte_stream_split, MD.Encoding.BYTE_STREAM_SPLIT), + (rle, MD.Encoding.RLE), + ) + for pageversion in (:v1, :v2), (input, encoding) in cases + checkencodedfile(input, encoding, pageversion) + end + + expected = Parquet._encodefile(delta_binary; encoding=MD.Encoding.DELTA_BINARY_PACKED) + @test expected == Parquet._encodefile(delta_binary; encoding=:DELTA_BINARY_PACKED) + @test expected == Parquet._encodefile(delta_binary; encoding="delta_binary_packed") + @test Parquet._encodefile(plain) == Parquet._encodefile(plain; encoding=nothing) +end + +@testset "empty explicit writer value payloads" begin + deltaempty = UInt8[0x80, 0x01, 0x04, 0x00, 0x00] + for values in (Int32[], Int64[], Union{Missing,Int32}[missing, missing], + Union{Missing,Int64}[missing, missing]) + @test encodedvaluepayload(values, MD.Encoding.DELTA_BINARY_PACKED) == deltaempty + end + for values in (String[], Vector{UInt8}[], Union{Missing,String}[missing, missing], + Union{Missing,Vector{UInt8}}[missing, missing]) + @test encodedvaluepayload(values, MD.Encoding.DELTA_LENGTH_BYTE_ARRAY) == deltaempty + @test encodedvaluepayload(values, MD.Encoding.DELTA_BYTE_ARRAY) == + vcat(deltaempty, deltaempty) + end + for T in (Int32, Int64, Float32, Float64) + @test isempty(encodedvaluepayload(T[], MD.Encoding.BYTE_STREAM_SPLIT)) + allnull = Union{Missing,T}[missing, missing] + @test isempty(encodedvaluepayload(allnull, MD.Encoding.BYTE_STREAM_SPLIT)) + end + @test encodedvaluepayload(Bool[], MD.Encoding.RLE) == zeros(UInt8, 4) + allnull = Union{Missing,Bool}[missing, missing] + @test encodedvaluepayload(allnull, MD.Encoding.RLE) == zeros(UInt8, 4) +end + +@testset "explicit writer value encoding validation" begin + physicals = ( + (MD.Type.BOOLEAN, (value=Bool[true],)), + (MD.Type.INT32, (value=Int32[1],)), + (MD.Type.INT64, (value=Int64[1],)), + (MD.Type.FLOAT, (value=Float32[1],)), + (MD.Type.DOUBLE, (value=Float64[1],)), + (MD.Type.BYTE_ARRAY, (value=["one"],)), + ) + encodings = ( + MD.Encoding.PLAIN, + MD.Encoding.DELTA_BINARY_PACKED, + MD.Encoding.DELTA_LENGTH_BYTE_ARRAY, + MD.Encoding.DELTA_BYTE_ARRAY, + MD.Encoding.BYTE_STREAM_SPLIT, + MD.Encoding.RLE, + ) + for encoding in encodings, (physical, input) in physicals + valid = encoding == MD.Encoding.PLAIN || + encoding == MD.Encoding.DELTA_BINARY_PACKED && + physical in (MD.Type.INT32, MD.Type.INT64) || + encoding in (MD.Encoding.DELTA_LENGTH_BYTE_ARRAY, MD.Encoding.DELTA_BYTE_ARRAY) && + physical == MD.Type.BYTE_ARRAY || + encoding == MD.Encoding.BYTE_STREAM_SPLIT && + physical in (MD.Type.INT32, MD.Type.INT64, MD.Type.FLOAT, MD.Type.DOUBLE) || + encoding == MD.Encoding.RLE && physical == MD.Type.BOOLEAN + valid || @test_throws ArgumentError Parquet._encodefile(input; encoding=encoding) + end + for encoding in (MD.Encoding.PLAIN_DICTIONARY, MD.Encoding.RLE_DICTIONARY, + MD.Encoding.BIT_PACKED, MD.Encoding.T(Int32(99)), :unknown, 5) + @test_throws ArgumentError Parquet._encodefile((value=Int32[1],); encoding=encoding) + end + @test_throws ArgumentError Parquet._encodefile((value=Int32[1],); + encoding=MD.Encoding.PLAIN, dictionary=true) + mixed = (valid=fill(Int32(1), 100), invalid=fill("x", 100)) + @test_throws ArgumentError Parquet._encodefile(mixed; + encoding=MD.Encoding.DELTA_BINARY_PACKED, + limits=Parquet.Limits(max_page_bytes=1)) + io = IOBuffer() + Parquet.write(io, (value=Int32[1],); encoding=:plain) + table = Parquet.Table(take!(io)) + @test table.columns.value == Int32[1] + close(table) +end + +@testset "public per-column writer encoding policy" begin + plain = (value=Int32[1, -2, 3],) + expected = Parquet._encodefile(plain) + @test expected == publicwritebytes(plain; encoding=nothing) + @test expected == publicwritebytes(plain; encoding=:PLAIN) + @test expected == publicwritebytes(plain; encoding="plain") + @test expected == publicwritebytes(plain; encoding=:value => :plain) + @test expected == publicwritebytes(plain; encoding="value" => "PLAIN") + @test expected == publicwritebytes(plain; encoding=(value=:plain,)) + @test expected == publicwritebytes(plain; encoding=Dict("value" => :plain)) + + repeated = (value=fill("same", 128),) + dictionary = Parquet._encodefile(repeated; dictionary=true) + @test dictionary == publicwritebytes(repeated; encoding=:dictionary) + @test dictionary == publicwritebytes(repeated; + encoding=:value => "DICTIONARY") + + count = 128 + input = ( + id=Int32.(1:count), + text=["prefix-$(index)" for index in 1:count], + score=Float64.(1:count) ./ 10, + active=[isodd(index) for index in 1:count], + fixed=fill((0x01, 0x02, 0x03, 0x04), count), + plainvalue=fill(Int64(7), count), + category=fill("category", count), + ) + policy = ( + id=:delta_binary_packed, + text="DELTA_BYTE_ARRAY", + score=:byte_stream_split, + active=:rle, + fixed=:byte_stream_split, + plainvalue=:plain, + ) + expectedencodings = ( + MD.Encoding.DELTA_BINARY_PACKED, + MD.Encoding.DELTA_BYTE_ARRAY, + MD.Encoding.BYTE_STREAM_SPLIT, + MD.Encoding.RLE, + MD.Encoding.BYTE_STREAM_SPLIT, + MD.Encoding.PLAIN, + ) + for pageversion in (:v1, :v2) + io = IOBuffer() + Parquet.write(io, input; dictionary=true, encoding=policy, + pageversion=pageversion) + bytes = take!(io) + table = Parquet.Table(bytes) + for name in keys(input) + expected = expectedtablevalues(getproperty(input, name)) + @test isequal(getproperty(table.columns, name), expected) + end + close(table) + for (index, encoding) in enumerate(expectedencodings) + pages, metadata = writtenpages(bytes, index) + @test metadata.encodings == [encoding] + @test metadata.dictionary_page_offset === nothing + @test length(pages) == 1 + end + pages, metadata = writtenpages(bytes, length(input)) + @test metadata.encodings == [MD.Encoding.PLAIN, MD.Encoding.RLE, + MD.Encoding.RLE_DICTIONARY] + @test metadata.dictionary_page_offset !== nothing + @test [page.header.type_ for page in pages] == + [MD.PageType.DICTIONARY_PAGE, + pageversion === :v1 ? MD.PageType.DATA_PAGE : MD.PageType.DATA_PAGE_V2] + @test metadata.encoding_stats == MD.PageEncodingStats[ + MD.PageEncodingStats(page_type=MD.PageType.DICTIONARY_PAGE, + encoding=MD.Encoding.PLAIN, count=Int32(1)), + MD.PageEncodingStats( + page_type=pageversion === :v1 ? MD.PageType.DATA_PAGE : + MD.PageType.DATA_PAGE_V2, + encoding=MD.Encoding.RLE_DICTIONARY, count=Int32(1)), + ] + end + + partial = (id=Int32[1, 2, 3], label=["a", "b", "c"]) + bytes = Parquet._encodefile(partial; encoding=(id=:delta_binary_packed,)) + @test writtenpages(bytes, 1)[2].encodings == [MD.Encoding.DELTA_BINARY_PACKED] + @test writtenpages(bytes, 2)[2].encodings == [MD.Encoding.PLAIN] + firstorder = Dict{Any,Any}(:id => :delta_binary_packed, :label => :delta_byte_array) + secondorder = Dict{Any,Any}(:label => :delta_byte_array, :id => :delta_binary_packed) + @test Parquet._encodefile(partial; encoding=firstorder) == + Parquet._encodefile(partial; encoding=secondorder) + + @test_throws ArgumentError Parquet._encodefile(plain; + encoding=(unknown=:plain,)) + duplicate = Dict{Any,Any}(:value => :plain, "value" => :plain) + @test_throws ArgumentError Parquet._encodefile(plain; encoding=duplicate) + @test Parquet._encodefile(plain; encoding=Dict(1 => :plain)) == + Parquet._encodefile(plain; encoding=(value=:plain,)) + @test_throws ArgumentError Parquet._encodefile(plain; encoding=(value=nothing,)) + @test_throws ArgumentError Parquet._encodefile(plain; encoding=(value=1,)) + @test_throws ArgumentError Parquet._encodefile((value=["one"],); + encoding=(value=:delta_binary_packed,)) + @test_throws ArgumentError Parquet._encodefile(plain; + dictionary=true, encoding=:plain) + @test_throws ArgumentError Parquet._encodefile(plain; + encoding=(value=:rle_dictionary,)) + @test_throws ArgumentError Parquet._encodefile(plain; + encoding=(value=:bit_packed,)) + @test_throws ArgumentError Parquet._encodefile(plain; encoding=(:plain,)) + + io = IOBuffer() + @test_throws ArgumentError Parquet.write(io, plain; encoding=(typo=:plain,)) + @test isempty(take!(io)) + mktempdir() do directory + path = joinpath(directory, "invalid.parquet") + @test_throws ArgumentError Parquet.write(path, plain; + encoding=(value=:delta_byte_array,)) + @test !ispath(path) + end +end + +@testset "FIXED_LEN_BYTE_ARRAY writer" begin + required = NTuple{3,UInt8}[ + (0x01, 0x02, 0x03), + (0x01, 0x02, 0x04), + (0x09, 0x08, 0x07), + ] + optional = Union{Missing,NTuple{3,UInt8}}[ + (0x01, 0x02, 0x03), + missing, + (0x09, 0x08, 0x07), + ] + input = (required=required, optional=optional) + for pageversion in (:v1, :v2), encoding in ( + MD.Encoding.PLAIN, MD.Encoding.DELTA_BYTE_ARRAY, + MD.Encoding.BYTE_STREAM_SPLIT) + checkencodedfile(input, encoding, pageversion) + bytes = Parquet._encodefile(input; encoding=encoding, pageversion=pageversion) + file = Parquet.File(bytes) + metadata = TH.decode(file.footer.bytes, MD.FileMetaData) + @test all(element -> element.type_ == MD.Type.FIXED_LEN_BYTE_ARRAY, + metadata.schema[2:end]) + @test all(element -> element.type_length == 3, metadata.schema[2:end]) + @test all(chunk -> chunk.meta_data.type_ == MD.Type.FIXED_LEN_BYTE_ARRAY, + metadata.row_groups[1].columns) + close(file) + end + + matrix = UInt8[0x01 0x01 0x09; 0x02 0x02 0x08; 0x03 0x04 0x07] + @test encodedvaluepayload(required, MD.Encoding.PLAIN) == collect(vec(matrix)) + @test encodedvaluepayload(required, MD.Encoding.DELTA_BYTE_ARRAY) == + Parquet.encode_delta_byte_array_fixed(matrix) + @test encodedvaluepayload(required, MD.Encoding.BYTE_STREAM_SPLIT) == + Parquet.encode_byte_stream_split_fixed(matrix) + + for pageversion in (:v1, :v2), encoding in ( + MD.Encoding.PLAIN, MD.Encoding.DELTA_BYTE_ARRAY, + MD.Encoding.BYTE_STREAM_SPLIT) + empty = NTuple{4,UInt8}[] + checkencodedfile((value=empty,), encoding, pageversion) + allnull = Union{Missing,NTuple{2,UInt8}}[missing, missing] + checkencodedfile((value=allnull,), encoding, pageversion) + end + @test isempty(encodedvaluepayload(NTuple{4,UInt8}[], MD.Encoding.PLAIN)) + @test isempty(encodedvaluepayload(NTuple{4,UInt8}[], MD.Encoding.BYTE_STREAM_SPLIT)) + @test encodedvaluepayload(NTuple{4,UInt8}[], MD.Encoding.DELTA_BYTE_ARRAY) == + Parquet.encode_delta_byte_array_fixed(Matrix{UInt8}(undef, 4, 0)) + + repeated = (value=fill((0x01, 0x02, 0x03, 0x04), 128),) + dictionary = Parquet._encodefile(repeated; dictionary=true) + _, metadata = writtenpages(dictionary, 1) + @test metadata.dictionary_page_offset !== nothing + @test metadata.encodings == [MD.Encoding.PLAIN, MD.Encoding.RLE, + MD.Encoding.RLE_DICTIONARY] + table = Parquet.Table(dictionary) + @test table.columns.value == expectedtablevalues(repeated.value) + close(table) + + @test_throws ArgumentError Parquet._encodefile((value=NTuple{0,UInt8}[()],)) + @test_throws ArgumentError Parquet._encodefile((value=[(UInt8(1), Int8(2))],)) + fixed = (value=NTuple{3,UInt8}[(0x01, 0x02, 0x03)],) + for encoding in (MD.Encoding.DELTA_BINARY_PACKED, + MD.Encoding.DELTA_LENGTH_BYTE_ARRAY, MD.Encoding.RLE) + @test_throws ArgumentError Parquet._encodefile(fixed; encoding=encoding) + end + @test_throws Parquet.LimitError Parquet._encodefile(fixed; + limits=Parquet.Limits(max_string_bytes=2)) + @test_throws Parquet.LimitError Parquet._encodefile(fixed; + limits=Parquet.Limits(max_page_bytes=2)) +end diff --git a/test/write_encoding_paths.jl b/test/write_encoding_paths.jl new file mode 100644 index 0000000..2a33733 --- /dev/null +++ b/test/write_encoding_paths.jl @@ -0,0 +1,128 @@ +using Parquet +using Test + +function selectorleaf(ordinal::Integer, path::Vector{String}, values::AbstractVector) + source = Parquet._writecolumn(Symbol(last(path)), values) + column = Parquet._withoutcolumnschema(source, path) + return Parquet.WriteLeafPlan(Int32(ordinal), copy(path), column) +end + +function assertencoding(choice::Parquet.WriteEncodingChoice, encoding; + dictionary::Bool=false) + @test choice.encoding == encoding + @test choice.dictionary == dictionary + return +end + +@testset "nested writer leaf encoding selectors" begin + MD = Parquet.Metadata + leaves = Parquet.WriteLeafPlan[ + selectorleaf(1, ["id"], Int32[1]), + selectorleaf(2, ["orders", "list", "element", "price"], Int64[2]), + selectorleaf(3, ["single", "value"], ["three"]), + selectorleaf(4, ["a.b"], Float64[4]), + selectorleaf(5, ["a", "b"], Int32[5]), + ] + + defaults = Parquet._writeencodingchoices(leaves, nothing, true) + @test length(defaults) == length(leaves) + @test all(choice -> choice.encoding === nothing && choice.dictionary, defaults) + + tablewide = Parquet._writeencodingchoices(leaves, :plain, false) + @test length(tablewide) == length(leaves) + @test all(choice -> choice.encoding == MD.Encoding.PLAIN && + !choice.dictionary, tablewide) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, :plain, true) + + policies = Dict{Any,Any}( + 1 => :delta_binary_packed, + (:orders, "list", :element, "price") => :delta_binary_packed, + :single => :dictionary, + "a.b" => :byte_stream_split, + ("a", :b) => :plain, + ) + choices = Parquet._writeencodingchoices(leaves, policies, false) + assertencoding(choices[1], MD.Encoding.DELTA_BINARY_PACKED) + assertencoding(choices[2], MD.Encoding.DELTA_BINARY_PACKED) + assertencoding(choices[3], nothing; dictionary=true) + assertencoding(choices[4], MD.Encoding.BYTE_STREAM_SPLIT) + assertencoding(choices[5], MD.Encoding.PLAIN) + + flat = Parquet._writeencodingchoices(leaves, :id => :plain, false) + assertencoding(flat[1], MD.Encoding.PLAIN) + alias = Parquet._writeencodingchoices(leaves, "single" => :dictionary, false) + assertencoding(alias[3], nothing; dictionary=true) + exact = Parquet._writeencodingchoices( + leaves, (:orders, :list, :element, :price) => :plain, false) + assertencoding(exact[2], MD.Encoding.PLAIN) + ordinal = Parquet._writeencodingchoices(leaves, Int32(5) => :plain, false) + assertencoding(ordinal[5], MD.Encoding.PLAIN) + + duplicatebare = Dict{Any,Any}(:id => :plain, "id" => :dictionary) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, duplicatebare, false) + duplicatepath = Dict{Any,Any}( + (:orders, :list, :element, :price) => :plain, + ("orders", "list", "element", "price") => :dictionary, + ) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, duplicatepath, false) + doubleassignment = Dict{Any,Any}( + 1 => :plain, + (:id,) => :delta_binary_packed, + ) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, doubleassignment, false) + + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, 0 => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, -1 => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, true => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, 99 => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, () => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, (:orders, 1) => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, 1.0 => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, :unknown => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, ("orders", "missing") => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + leaves, (:single, :value) => :delta_binary_packed, false) + + multidotted = Parquet.WriteLeafPlan[ + selectorleaf(1, ["a", "b"], Int32[1]), + ] + @test_throws ArgumentError Parquet._writeencodingchoices( + multidotted, "a.b" => :plain, false) + + multigroup = Parquet.WriteLeafPlan[ + selectorleaf(1, ["pair", "left"], Int32[1]), + selectorleaf(2, ["pair", "right"], Int32[2]), + ] + @test_throws ArgumentError Parquet._writeencodingchoices( + multigroup, :pair => :plain, false) + pairchoices = Parquet._writeencodingchoices( + multigroup, (:pair, :left) => :delta_binary_packed, true) + assertencoding(pairchoices[1], MD.Encoding.DELTA_BINARY_PACKED) + assertencoding(pairchoices[2], nothing; dictionary=true) + + duplicatepaths = Parquet.WriteLeafPlan[ + selectorleaf(6, ["duplicate", "value"], Int32[1]), + selectorleaf(7, ["duplicate", "value"], Int32[2]), + ] + @test_throws ArgumentError Parquet._writeencodingchoices( + duplicatepaths, (:duplicate, :value) => :plain, false) + @test_throws ArgumentError Parquet._writeencodingchoices( + duplicatepaths, :duplicate => :plain, false) + duplicateordinal = Parquet._writeencodingchoices( + duplicatepaths, 7 => :delta_binary_packed, false) + assertencoding(duplicateordinal[1], nothing) + assertencoding(duplicateordinal[2], MD.Encoding.DELTA_BINARY_PACKED) +end diff --git a/test/write_logical.jl b/test/write_logical.jl new file mode 100644 index 0000000..f64a742 --- /dev/null +++ b/test/write_logical.jl @@ -0,0 +1,423 @@ +using Dates +using Test +using UUIDs + +function rewritelogicalschema(replacement::Function, bytes::Vector{UInt8}, + index::Int=2) + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + prefix = bytes[1:Int(file.footer.offset)] + close(file) + schema = copy(metadata.schema) + schema[index] = replacement(schema[index]) + updated = Parquet.Metadata.FileMetaData( + version=metadata.version, + schema=schema, + num_rows=metadata.num_rows, + row_groups=metadata.row_groups, + key_value_metadata=metadata.key_value_metadata, + created_by=metadata.created_by, + column_orders=metadata.column_orders, + encryption_algorithm=metadata.encryption_algorithm, + footer_signing_key_metadata=metadata.footer_signing_key_metadata, + unknown_fields=metadata.unknown_fields, + ) + footer = Parquet.Thrift.encode(updated) + output = vcat(prefix, footer) + Parquet._writelittle!(output, UInt32(length(footer))) + append!(output, Parquet.PARQUET_MAGIC) + return output +end + +function withlogical(element::Parquet.Metadata.SchemaElement; logical, + converted=element.converted_type, precision=element.precision, + scale=element.scale) + return Parquet.Metadata.SchemaElement( + type_=element.type_, + type_length=element.type_length, + repetition_type=element.repetition_type, + name=element.name, + num_children=element.num_children, + converted_type=converted, + scale=scale, + precision=precision, + field_id=element.field_id, + logicalType=logical, + unknown_fields=element.unknown_fields, + ) +end + +@testset "temporal and integer writer round trips" begin + times = Union{Missing,Time}[ + Time(0), missing, Time(23, 59, 59, 999, 999, 999)] + datetimes = DateTime[ + DateTime(1969, 12, 31, 23, 59, 59, 999), + DateTime(1970, 1, 1), + DateTime(2000, 2, 29, 12, 34, 56, 789), + ] + micros = Union{Missing,Parquet.Timestamp{:micros}}[ + Parquet.Timestamp(typemin(Int64), :micros, true), + missing, + Parquet.Timestamp(typemax(Int64), :micros, true), + ] + nanos = Parquet.Timestamp{:nanos}[ + Parquet.Timestamp(-1, :nanos, false), + Parquet.Timestamp(0, :nanos, false), + Parquet.Timestamp(1, :nanos, false), + ] + millis = Parquet.Timestamp{:millis}[ + Parquet.Timestamp(typemin(Int64), :millis, true), + Parquet.Timestamp(0, :millis, true), + Parquet.Timestamp(typemax(Int64), :millis, true), + ] + input = ( + i8=Int8[-128, 0, 127], + u8=UInt8[0, 127, 255], + i16=Int16[-32768, 0, 32767], + u16=UInt16[0, 32767, 65535], + u32=UInt32[0, 0x80000000, 0xffffffff], + u64=UInt64[0, 0x8000000000000000, 0xffffffffffffffff], + times=times, + datetimes=datetimes, + micros=micros, + nanos=nanos, + millis=millis, + ) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion, + encoding=:delta_binary_packed) + table = Parquet.Table(bytes) + for name in keys(input) + @test isequal(getproperty(table.columns, name), getproperty(input, name)) + end + close(table) + + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + schema = metadata.schema + @test all(element -> element.logicalType.INTEGER !== nothing, schema[2:7]) + @test schema[8].logicalType.TIME.unit.NANOS !== nothing + @test !schema[8].logicalType.TIME.isAdjustedToUTC + @test schema[8].converted_type === nothing + @test schema[9].logicalType.TIMESTAMP.unit.MILLIS !== nothing + @test !schema[9].logicalType.TIMESTAMP.isAdjustedToUTC + @test schema[9].converted_type == + Parquet.Metadata.ConvertedType.TIMESTAMP_MILLIS + @test schema[10].logicalType.TIMESTAMP.unit.MICROS !== nothing + @test schema[10].logicalType.TIMESTAMP.isAdjustedToUTC + @test schema[10].converted_type == + Parquet.Metadata.ConvertedType.TIMESTAMP_MICROS + @test schema[11].logicalType.TIMESTAMP.unit.NANOS !== nothing + @test !schema[11].logicalType.TIMESTAMP.isAdjustedToUTC + @test schema[11].converted_type === nothing + @test schema[12].logicalType.TIMESTAMP.unit.MILLIS !== nothing + @test schema[12].logicalType.TIMESTAMP.isAdjustedToUTC + @test schema[12].converted_type == + Parquet.Metadata.ConvertedType.TIMESTAMP_MILLIS + end + + @test_throws ArgumentError Parquet._encodefile( + (value=Parquet.Timestamp{:micros}[],)) + @test_throws ArgumentError Parquet._encodefile((value=[ + Parquet.Timestamp(0, :nanos, true), + Parquet.Timestamp(1, :nanos, false), + ],)) +end + +@testset "binary logical writer round trips" begin + uuids = Union{Missing,UUID}[ + UUID("00112233-4455-6677-8899-aabbccddeeff"), + missing, + UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"), + ] + halves = Union{Missing,Float16}[ + reinterpret(Float16, UInt16(0x0000)), + reinterpret(Float16, UInt16(0x8000)), + reinterpret(Float16, UInt16(0x7e01)), + ] + json = Union{Missing,Parquet.JSONValue}[ + Parquet.JSONValue(codeunits("{\"a\":1}")), + missing, + Parquet.JSONValue(codeunits("[1,null,3]")), + ] + bson = Union{Missing,Parquet.BSONValue}[ + Parquet.BSONValue(hex2bytes("0c0000001061000100000000")), + missing, + Parquet.BSONValue(hex2bytes("090000000a00ff0000")), + ] + intervals = Union{Missing,Parquet.Interval}[ + Parquet.Interval(1, 2, 3), + missing, + Parquet.Interval(typemax(UInt32), 0, 86_400_000), + ] + nulls = Missing[missing, missing, missing] + input = (; uuids, halves, json, bson, intervals, nulls) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion, codec=:zstd) + table = Parquet.Table(bytes) + @test isequal(table.columns.uuids, uuids) + @test reinterpret(UInt16, collect(skipmissing(table.columns.halves))) == + reinterpret(UInt16, collect(skipmissing(halves))) + @test isequal(table.columns.json, json) + @test isequal(table.columns.bson, bson) + @test isequal(table.columns.intervals, intervals) + @test isequal(table.columns.nulls, nulls) + close(table) + + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + schema = metadata.schema + @test schema[2].logicalType.UUID !== nothing + @test schema[2].type_length == 16 + @test schema[3].logicalType.FLOAT16 !== nothing + @test schema[3].type_length == 2 + @test schema[4].logicalType.JSON !== nothing + @test schema[4].converted_type == Parquet.Metadata.ConvertedType.JSON + @test schema[5].logicalType.BSON !== nothing + @test schema[5].converted_type == Parquet.Metadata.ConvertedType.BSON + @test schema[6].logicalType === nothing + @test schema[6].converted_type == Parquet.Metadata.ConvertedType.INTERVAL + @test schema[6].type_length == 12 + @test schema[7].logicalType.UNKNOWN !== nothing + @test schema[7].type_ == Parquet.Metadata.Type.INT32 + end +end + +@testset "DECIMAL writer inference and round trips" begin + small = Union{Missing,Parquet.Decimal}[ + Parquet.Decimal(12345, 2), missing, Parquet.Decimal(-99999, 2)] + medium = Parquet.Decimal[ + Parquet.Decimal(123456789012345678, 6), + Parquet.Decimal(-1, 6), + Parquet.Decimal(0, 6), + ] + wide = Parquet.Decimal[ + Parquet.Decimal(big"12345678901234567890", 4), + Parquet.Decimal(big"-12345678901234567890", 4), + Parquet.Decimal(0, 4), + ] + input = (; small, medium, wide) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion, + encoding=(small=:delta_binary_packed, medium=:delta_binary_packed, + wide=:delta_byte_array)) + table = Parquet.Table(bytes) + @test isequal(table.columns.small, small) + @test table.columns.medium == medium + @test table.columns.wide == wide + close(table) + + file = Parquet.File(bytes) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test [element.type_ for element in metadata.schema[2:end]] == [ + Parquet.Metadata.Type.INT32, + Parquet.Metadata.Type.INT64, + Parquet.Metadata.Type.FIXED_LEN_BYTE_ARRAY, + ] + @test [element.precision for element in metadata.schema[2:end]] == + Int32[5, 18, 20] + @test [element.scale for element in metadata.schema[2:end]] == + Int32[2, 6, 4] + @test all(element -> element.logicalType.DECIMAL !== nothing, + metadata.schema[2:end]) + @test all(element -> element.converted_type == + Parquet.Metadata.ConvertedType.DECIMAL, metadata.schema[2:end]) + end + + @test_throws ArgumentError Parquet._encodefile( + (value=Parquet.Decimal[],)) + @test_throws ArgumentError Parquet._encodefile( + (value=Union{Missing,Parquet.Decimal}[missing],)) + @test_throws ArgumentError Parquet._encodefile( + (value=[Parquet.Decimal(1, 1), Parquet.Decimal(1, 2)],)) +end + +@testset "schema-bearing Table exact logical rewrite" begin + timestampbytes = Parquet._encodefile( + (value=DateTime[DateTime(1970), DateTime(2000, 1, 1)],)) + timestampbytes = rewritelogicalschema(timestampbytes) do element + logical = Parquet.Metadata.LogicalType( + TIMESTAMP=Parquet.Metadata.TimestampType( + isAdjustedToUTC=true, + unit=Parquet.Metadata.TimeUnit( + MILLIS=Parquet.Metadata.MilliSeconds()), + ), + ) + return withlogical(element; logical=logical, + converted=Parquet.Metadata.ConvertedType.TIME_MICROS) + end + table = Parquet.Table(timestampbytes) + rewritten = Parquet._encodefile(table) + close(table) + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test metadata.schema[2].logicalType.TIMESTAMP.isAdjustedToUTC + @test metadata.schema[2].converted_type == + Parquet.Metadata.ConvertedType.TIME_MICROS + + timebytes = Parquet._encodefile( + (value=Time[Time(Dates.Nanosecond(1_000))],)) + timebytes = rewritelogicalschema(timebytes) do element + logical = Parquet.Metadata.LogicalType( + TIME=Parquet.Metadata.TimeType( + isAdjustedToUTC=true, + unit=Parquet.Metadata.TimeUnit( + MICROS=Parquet.Metadata.MicroSeconds()), + ), + ) + return withlogical(element; logical=logical, + converted=Parquet.Metadata.ConvertedType.TIME_MICROS) + end + table = Parquet.Table(timebytes) + @test table.columns.value == [Time(Dates.Nanosecond(1_000_000))] + rewritten = Parquet._encodefile(table) + close(table) + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test metadata.schema[2].logicalType.TIME.unit.MICROS !== nothing + @test metadata.schema[2].logicalType.TIME.isAdjustedToUTC + @test metadata.schema[2].converted_type == Parquet.Metadata.ConvertedType.TIME_MICROS + + decimalbytes = Parquet._encodefile( + (value=Parquet.Decimal[Parquet.Decimal(12345, 2)],)) + decimalbytes = rewritelogicalschema(decimalbytes) do element + logical = Parquet.Metadata.LogicalType( + DECIMAL=Parquet.Metadata.DecimalType(scale=Int32(2), precision=Int32(9))) + return withlogical(element; logical=logical, + converted=Parquet.Metadata.ConvertedType.DATE, + precision=Int32(99), scale=Int32(99)) + end + table = Parquet.Table(decimalbytes) + rewritten = Parquet._encodefile(table) + close(table) + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test metadata.schema[2].precision == 99 + @test metadata.schema[2].scale == 99 + @test metadata.schema[2].logicalType.DECIMAL.precision == 9 + @test metadata.schema[2].logicalType.DECIMAL.scale == 2 + @test metadata.schema[2].converted_type == Parquet.Metadata.ConvertedType.DATE + + enumbytes = Parquet._encodefile((value=["alpha", "beta"],)) + enumbytes = rewritelogicalschema(enumbytes) do element + logical = Parquet.Metadata.LogicalType(ENUM=Parquet.Metadata.EnumType()) + return withlogical(element; logical=logical, + converted=Parquet.Metadata.ConvertedType.UTF8) + end + table = Parquet.Table(enumbytes) + @test table.columns.value == ["alpha", "beta"] + rewritten = Parquet._encodefile(table) + close(table) + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test metadata.schema[2].logicalType.ENUM !== nothing + @test metadata.schema[2].converted_type == Parquet.Metadata.ConvertedType.UTF8 + + legacystring = Parquet._encodefile((value=["legacy"],)) + legacystring = rewritelogicalschema(legacystring) do element + return withlogical(element; logical=nothing, + converted=Parquet.Metadata.ConvertedType.UTF8) + end + table = Parquet.Table(legacystring) + @test table.columns.value == ["legacy"] + rewritten = Parquet._encodefile(table) + close(table) + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test metadata.schema[2].logicalType === nothing + @test metadata.schema[2].converted_type == Parquet.Metadata.ConvertedType.UTF8 + + modernstring = rewritelogicalschema(legacystring) do element + logical = Parquet.Metadata.LogicalType(STRING=Parquet.Metadata.StringType()) + return withlogical(element; logical=logical, + converted=Parquet.Metadata.ConvertedType.DATE) + end + table = Parquet.Table(modernstring) + rewritten = Parquet._encodefile(table) + close(table) + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test metadata.schema[2].logicalType.STRING !== nothing + @test metadata.schema[2].converted_type == Parquet.Metadata.ConvertedType.DATE + + raw = (Parquet.Thrift.RawField(42, Parquet.Thrift.I32, UInt8[0x02]),) + source = Parquet.Metadata.SchemaElement( + type_=Parquet.Metadata.Type.BYTE_ARRAY, + type_length=Int32(7), + repetition_type=Parquet.Metadata.FieldRepetitionType.OPTIONAL, + name="preserved", + num_children=Int32(0), + converted_type=Parquet.Metadata.ConvertedType.DATE, + scale=Int32(9), + precision=Int32(9), + field_id=Int32(17), + logicalType=Parquet.Metadata.LogicalType( + STRING=Parquet.Metadata.StringType()), + unknown_fields=raw, + ) + canonical = Parquet._canonicalwriteelement(source) + @test canonical.type_ == source.type_ + @test canonical.type_length == source.type_length + @test canonical.repetition_type == source.repetition_type + @test canonical.name == source.name + @test canonical.num_children == source.num_children + @test canonical.field_id == source.field_id + @test canonical.unknown_fields == source.unknown_fields + @test canonical.scale === nothing + @test canonical.precision === nothing + @test canonical.converted_type == Parquet.Metadata.ConvertedType.UTF8 + + opaquebytes = Parquet._encodefile( + (value=Vector{UInt8}[collect(codeunits("future"))],)) + opaque = Parquet.Metadata.LogicalType(unknown_fields=( + Parquet.Thrift.RawField(127, Parquet.Thrift.STRUCT, UInt8[0x00]), + )) + opaquebytes = rewritelogicalschema(opaquebytes) do element + return withlogical(element; logical=opaque, + converted=Parquet.Metadata.ConvertedType.UTF8) + end + table = Parquet.Table(opaquebytes) + @test table.columns.value == Vector{UInt8}[collect(codeunits("future"))] + rewritten = Parquet._encodefile(table) + close(table) + file = Parquet.File(rewritten) + metadata = Parquet.Thrift.decode(file.footer.bytes, Parquet.Metadata.FileMetaData) + close(file) + @test metadata.schema[2].logicalType == opaque + @test metadata.schema[2].converted_type == Parquet.Metadata.ConvertedType.UTF8 +end + +@testset "unsupported temporal units stay explicit at the Table boundary" begin + bytes = Parquet._encodefile((value=Int64[1],)) + unknownunit = Parquet.Metadata.TimeUnit(unknown_fields=( + Parquet.Thrift.RawField(42, Parquet.Thrift.STRUCT, UInt8[0x00]), + )) + unknown = rewritelogicalschema(bytes) do element + logical = Parquet.Metadata.LogicalType( + TIMESTAMP=Parquet.Metadata.TimestampType( + isAdjustedToUTC=true, unit=unknownunit)) + return withlogical(element; logical=logical) + end + file = Parquet.File(unknown) + close(file) + @test_throws Parquet.UnsupportedFeatureError Parquet.Table(unknown) + + empty = rewritelogicalschema(bytes) do element + logical = Parquet.Metadata.LogicalType( + TIMESTAMP=Parquet.Metadata.TimestampType( + isAdjustedToUTC=true, unit=Parquet.Metadata.TimeUnit())) + return withlogical(element; logical=logical) + end + file = Parquet.File(empty) + close(file) + @test_throws Parquet.FormatError Parquet.Table(empty) +end diff --git a/test/write_nested.jl b/test/write_nested.jl new file mode 100644 index 0000000..db4a890 --- /dev/null +++ b/test/write_nested.jl @@ -0,0 +1,1377 @@ +using Dates +using Tables +using Test + +const WNMD = Parquet.Metadata + +struct WNDeclaredVector{T} <: AbstractVector{T} + values::Vector{Any} +end + +function Base.IndexStyle(::Type{<:WNDeclaredVector}) + return IndexLinear() +end + +function Base.size(values::WNDeclaredVector) + return size(values.values) +end + +function Base.getindex(values::WNDeclaredVector, index::Int) + return values.values[index] +end + +struct WNBadMap <: AbstractDict{String,Int32} end + +function Base.length(::WNBadMap) + return 1 +end + +function Base.getindex(::WNBadMap, ::String) + return Int32(1) +end + +function Base.iterate(::WNBadMap, state::Bool=false) + state && return nothing + return missing => Int32(1), true +end + +mutable struct WNTrackedRows{S} + consumed::Base.RefValue{Int} + rows::Int +end + +Tables.istable(::Type{<:WNTrackedRows}) = true +Tables.rowaccess(::Type{<:WNTrackedRows}) = true +Tables.rows(rows::WNTrackedRows) = rows +Tables.schema(::WNTrackedRows) = Tables.Schema((:value,), (Int32,)) +Base.IteratorSize(::Type{WNTrackedRows{S}}) where {S} = S() +Base.length(rows::WNTrackedRows{Base.HasLength}) = rows.rows + +function Base.iterate(rows::WNTrackedRows, state::Int=1) + state > rows.rows && return nothing + rows.consumed[] += 1 + return (value=Int32(state),), state + 1 +end + +mutable struct WNFreshStringRows{S} + consumed::Base.RefValue{Int} + rows::Int + width::Int +end + +Tables.istable(::Type{<:WNFreshStringRows}) = true +Tables.rowaccess(::Type{<:WNFreshStringRows}) = true +Tables.rows(rows::WNFreshStringRows) = rows +Tables.schema(::WNFreshStringRows) = Tables.Schema((:value,), (String,)) +Base.IteratorSize(::Type{WNFreshStringRows{S}}) where {S} = S() +Base.length(rows::WNFreshStringRows{Base.HasLength}) = rows.rows + +function Base.iterate(rows::WNFreshStringRows, state::Int=1) + state > rows.rows && return nothing + rows.consumed[] += 1 + return (value=repeat("x", rows.width),), state + 1 +end + +mutable struct WNFreshPayloadRows{T,F} + consumed::Base.RefValue{Int} + rows::Int + width::Int + maker::F +end + +Tables.istable(::Type{<:WNFreshPayloadRows}) = true +Tables.rowaccess(::Type{<:WNFreshPayloadRows}) = true +Tables.rows(rows::WNFreshPayloadRows) = rows +Tables.schema(::WNFreshPayloadRows{T}) where {T} = + Tables.Schema((:value,), (T,)) +Base.IteratorSize(::Type{<:WNFreshPayloadRows}) = Base.HasLength() +Base.length(rows::WNFreshPayloadRows) = rows.rows + +function Base.iterate(rows::WNFreshPayloadRows, state::Int=1) + state > rows.rows && return nothing + rows.consumed[] += 1 + return (value=rows.maker(rows.width),), state + 1 +end + +function wnfreshjson(width::Int) + return Parquet.JSONValue(codeunits(string('"', repeat("x", width), '"'))) +end + +function wnfreshbson(width::Int) + total = width + 13 + bytes = UInt8[ + UInt8(total & 0xff), + UInt8((total >> 8) & 0xff), + UInt8((total >> 16) & 0xff), + UInt8((total >> 24) & 0xff), + 0x05, 0x78, 0x00, + UInt8(width & 0xff), + UInt8((width >> 8) & 0xff), + UInt8((width >> 16) & 0xff), + UInt8((width >> 24) & 0xff), + 0x00, + ] + append!(bytes, fill(UInt8(1), width)) + push!(bytes, 0x00) + return Parquet.BSONValue(bytes) +end + +function wnfreshdecimal(width::Int) + return Parquet.Decimal(big(1) << (8 * width), 0) +end + +struct WNSchemaLessRows{T} + values::Vector{T} +end + +Tables.istable(::Type{<:WNSchemaLessRows}) = true +Tables.rowaccess(::Type{<:WNSchemaLessRows}) = true +Tables.rows(rows::WNSchemaLessRows) = rows +Base.IteratorSize(::Type{<:WNSchemaLessRows}) = Base.HasLength() +Base.IteratorEltype(::Type{<:WNSchemaLessRows}) = Base.HasEltype() +Base.eltype(::Type{WNSchemaLessRows{T}}) where {T} = T +Base.length(rows::WNSchemaLessRows) = length(rows.values) +Base.iterate(rows::WNSchemaLessRows, state...) = iterate(rows.values, state...) + +struct WNZeroVector{T} <: AbstractVector{T} + values::Vector{T} +end + +Base.IndexStyle(::Type{<:WNZeroVector}) = IndexCartesian() +Base.size(values::WNZeroVector) = size(values.values) +Base.axes(values::WNZeroVector) = (0:(length(values.values) - 1),) +Base.getindex(values::WNZeroVector, index::Int) = values.values[index + 1] + +mutable struct WNThrowingSizeVector{T,E} <: AbstractVector{T} + values::Vector{T} + calls::Int + throw_on::Int + exception::E +end + +function Base.IndexStyle(::Type{<:WNThrowingSizeVector}) + return IndexLinear() +end + +function Base.size(values::WNThrowingSizeVector) + values.calls += 1 + values.calls == values.throw_on && throw(values.exception) + return size(values.values) +end + +function Base.length(values::WNThrowingSizeVector) + return length(values.values) +end + +function Base.axes(values::WNThrowingSizeVector) + return (Base.OneTo(length(values.values)),) +end + +function Base.firstindex(::WNThrowingSizeVector) + return 1 +end + +function Base.lastindex(values::WNThrowingSizeVector) + return length(values.values) +end + +function Base.getindex(values::WNThrowingSizeVector, index::Int) + return values.values[index] +end + +function wndeepstruct(depth::Int, rows::Int) + values::AbstractVector = iszero(rows) ? Int32[] : Int32[7] + for level in depth:-1:1 + childname = level == depth ? "value" : "level_$(level + 1)" + values = Parquet.StructVector(String[childname], + AbstractVector[values]; rows=rows) + end + return values +end + +function wnwriterpassallocations(source::AbstractVector) + limits = Parquet.Limits(max_materialized_bytes=1_000_000_000) + budget = Parquet._LiveByteBudget(limits) + shape = Parquet._nestedwriteshape("value", eltype(source), source, + limits, budget) + shapes = Parquet._NestedWriteShape[shape] + values = AbstractVector[source] + rows = length(source) + fragment = Parquet._nestedwriteschema(shape, limits, budget) + fragments = Vector{Parquet.Metadata.SchemaElement}[fragment] + semantic, plans = Parquet._nestedwritecompile(shapes, fragments, limits, + budget) + Parquet._nestedwritescanaggregates!(shapes, values, rows, limits, nothing) + scanbytes = @allocated Parquet._nestedwritescanaggregates!(shapes, values, + rows, limits, nothing) + counts = Parquet._nestedwritecounts(length(semantic.leaves), budget) + context = Parquet._NestedWriteCountContext(counts, limits) + Parquet._nestedwritescanrows!(context, plans, values, rows) + rowbytes = @allocated Parquet._nestedwritescanrows!(context, plans, + values, rows) + return scanbytes, rowbytes +end + +struct WNTrackedBytes <: AbstractVector{UInt8} + values::Vector{UInt8} + reads::Base.RefValue{Int} +end + +Base.IndexStyle(::Type{WNTrackedBytes}) = IndexLinear() +Base.size(values::WNTrackedBytes) = size(values.values) + +function Base.getindex(values::WNTrackedBytes, index::Int) + values.reads[] += 1 + return values.values[index] +end + +mutable struct WNChangingStrings <: AbstractVector{String} + calls::Base.RefValue{Int} + small::String + large::String +end + +Base.IndexStyle(::Type{WNChangingStrings}) = IndexLinear() +Base.size(::WNChangingStrings) = (1,) + +function Base.getindex(values::WNChangingStrings, ::Int) + values.calls[] += 1 + return values.calls[] <= 2 ? values.small : values.large +end + +function wnmutationattack(large::String) + source = WNChangingStrings(Ref(0), "x", large) + limits = Parquet.Limits(max_materialized_bytes=50_000) + budget = Parquet._LiveByteBudget(limits) + try + Parquet._writefields((value=source,), limits, budget) + error("expected nested writer mutation rejection") + catch err + err isa ArgumentError || rethrow() + end + @assert source.calls[] == 3 + @assert Parquet._budgetused(budget) == 0 + return +end + +function wninspect(bytes::Vector{UInt8}) + file = Parquet.File(bytes) + try + metadata = Parquet.Thrift.decode(copy(file.footer.bytes), + WNMD.FileMetaData) + schema = Parquet.Schema(metadata.schema) + streams = Parquet.LeafStream[] + if !isempty(metadata.row_groups) + rows = only(metadata.row_groups).num_rows + for index in eachindex(schema.leaves) + push!(streams, Parquet.readleafstream(file, metadata, schema, + 1, index; expected_rows=rows)) + end + end + return (metadata=metadata, schema=schema, streams=streams) + finally + close(file) + end +end + +function wnexpectstream(stream::Parquet.LeafStream, repetition, + definition, values; max_repetition::Integer, + max_definition::Integer) + @test stream.repetition == UInt64[repetition...] + @test stream.definition == UInt64[definition...] + @test stream.values == values + @test all(level -> level <= max_repetition, stream.repetition) + @test all(level -> level <= max_definition, stream.definition) + @test all(pair -> first(pair) <= last(pair), + zip(stream.repetition, stream.definition)) + return +end + +function wnbytes(values::AbstractVector{<:AbstractString}) + return Vector{UInt8}[collect(codeunits(value)) for value in values] +end + +function wnheaders(bytes::Vector{UInt8}, column::Int) + file = Parquet.File(bytes) + try + metadata = Parquet.Thrift.decode(copy(file.footer.bytes), + WNMD.FileMetaData) + chunk = only(metadata.row_groups).columns[column].meta_data + start, stop = Parquet._chunkrange(chunk, file.footer.offset) + headers = WNMD.PageHeader[] + position = start + while position < stop + frame = Parquet.readpage(file.source, position, stop, + Parquet.Limits()) + if frame.header.data_page_header !== nothing || + frame.header.data_page_header_v2 !== nothing + push!(headers, frame.header) + end + position = Parquet.pageend(frame) + end + return headers + finally + close(file) + end +end + +function wncheckpagetype(bytes::Vector{UInt8}, column::Int, + pageversion::Symbol, entries::Int, rows::Int, nulls::Int) + headers = wnheaders(bytes, column) + @test length(headers) == 1 + header = only(headers) + if pageversion === :v1 + @test header.type_ == WNMD.PageType.DATA_PAGE + @test header.data_page_header.num_values == entries + else + @test header.type_ == WNMD.PageType.DATA_PAGE_V2 + page = header.data_page_header_v2 + @test page.num_values == entries + @test page.num_rows == rows + @test page.num_nulls == nulls + end + return +end + +function wnchecklist(column, expected) + @test length(column) == length(expected) + for index in eachindex(expected) + row = expected[index] + if ismissing(row) + @test column[index] === missing + else + @test isequal(collect(column[index]), row) + end + end + return +end + +@testset "recursive writer optional struct" begin + S = NamedTuple{(:a,:b),Tuple{Int32,Union{Missing,String}}} + values = Union{Missing,S}[ + missing, + S((Int32(1), missing)), + S((Int32(2), "x")), + ] + input = (s=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == + ["schema", "s", "a", "b"] + @test result.metadata.schema[1].num_children == 1 + @test result.metadata.schema[2].repetition_type == + WNMD.FieldRepetitionType.OPTIONAL + @test result.metadata.schema[2].num_children == 2 + @test result.metadata.schema[3].type_ == WNMD.Type.INT32 + @test result.metadata.schema[3].repetition_type == + WNMD.FieldRepetitionType.REQUIRED + @test result.metadata.schema[4].type_ == WNMD.Type.BYTE_ARRAY + @test result.metadata.schema[4].repetition_type == + WNMD.FieldRepetitionType.OPTIONAL + @test result.metadata.schema[4].logicalType.STRING !== nothing + @test result.metadata.schema[4].converted_type == + WNMD.ConvertedType.UTF8 + @test [leaf.path for leaf in result.schema.leaves] == + [["s", "a"], ["s", "b"]] + wnexpectstream(result.streams[1], [0,0,0], [0,1,1], + Int32[1,2]; max_repetition=0, max_definition=1) + wnexpectstream(result.streams[2], [0,0,0], [0,1,2], + wnbytes(["x"]); max_repetition=0, max_definition=2) + table = Parquet.Table(bytes) + try + @test table.columns.s[1] === missing + @test table.columns.s[2]["a"] == 1 + @test table.columns.s[2]["b"] === missing + @test table.columns.s[3]["a"] == 2 + @test table.columns.s[3]["b"] == "x" + finally + close(table) + end + end +end + +@testset "recursive writer canonical LIST" begin + E = Union{Missing,Int32} + values = Union{Missing,Vector{E}}[ + missing, + E[], + E[missing,10,20], + E[30], + ] + input = (items=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == + ["schema", "items", "list", "element"] + outer, repeated, element = result.metadata.schema[2:4] + @test outer.repetition_type == WNMD.FieldRepetitionType.OPTIONAL + @test outer.logicalType.LIST !== nothing + @test outer.converted_type == WNMD.ConvertedType.LIST + @test outer.num_children == 1 + @test repeated.repetition_type == WNMD.FieldRepetitionType.REPEATED + @test repeated.num_children == 1 + @test repeated.logicalType === nothing + @test repeated.converted_type === nothing + @test element.type_ == WNMD.Type.INT32 + @test element.repetition_type == WNMD.FieldRepetitionType.OPTIONAL + @test only(result.schema.leaves).path == + ["items", "list", "element"] + wnexpectstream(only(result.streams), [0,0,0,1,1,0], + [0,1,2,3,3,3], Int32[10,20,30]; + max_repetition=1, max_definition=3) + wncheckpagetype(bytes, 1, pageversion, 6, 4, 3) + table = Parquet.Table(bytes) + try + wnchecklist(table.columns.items, values) + finally + close(table) + end + end +end + +@testset "recursive writer canonical MAP" begin + values = Parquet.MapVector( + Int32[0,0,0,2,3], + String["a","b","c"], + Union{Missing,Int32}[missing,2,3]; + validity=Bool[false,true,true,true], + ) + input = (attrs=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == + ["schema", "attrs", "key_value", "key", "value"] + outer, entries, key, value = result.metadata.schema[2:5] + @test outer.repetition_type == WNMD.FieldRepetitionType.OPTIONAL + @test outer.logicalType.MAP !== nothing + @test outer.converted_type == WNMD.ConvertedType.MAP + @test entries.repetition_type == WNMD.FieldRepetitionType.REPEATED + @test entries.logicalType === nothing + @test entries.converted_type === nothing + @test key.type_ == WNMD.Type.BYTE_ARRAY + @test key.repetition_type == WNMD.FieldRepetitionType.REQUIRED + @test key.logicalType.STRING !== nothing + @test value.type_ == WNMD.Type.INT32 + @test value.repetition_type == WNMD.FieldRepetitionType.OPTIONAL + @test [leaf.path for leaf in result.schema.leaves] == [ + ["attrs", "key_value", "key"], + ["attrs", "key_value", "value"], + ] + wnexpectstream(result.streams[1], [0,0,0,1,0], + [0,1,2,2,2], wnbytes(["a","b","c"]); + max_repetition=1, max_definition=2) + wnexpectstream(result.streams[2], [0,0,0,1,0], + [0,1,2,3,3], Int32[2,3]; + max_repetition=1, max_definition=3) + wncheckpagetype(bytes, 1, pageversion, 5, 4, 2) + wncheckpagetype(bytes, 2, pageversion, 5, 4, 3) + table = Parquet.Table(bytes) + try + column = table.columns.attrs + @test column[1] === missing + @test isempty(column[2]) + @test isequal(collect(column[3]), + Pair{String,Union{Missing,Int32}}["a" => missing, "b" => 2]) + @test collect(column[4]) == ["c" => Int32(3)] + finally + close(table) + end + end + + DictType = Dict{String,Union{Missing,Int32}} + dictionaries = DictType[ + DictType(), + DictType("a" => Int32(1), "b" => missing), + ] + table = Parquet.Table(Parquet._encodefile((attrs=dictionaries,))) + try + @test isempty(table.columns.attrs[1]) + @test isequal(Dict(table.columns.attrs[2]), dictionaries[2]) + finally + close(table) + end +end + +@testset "recursive writer LIST of optional structs" begin + T = NamedTuple{(:id,:label),Tuple{Int32,Union{Missing,String}}} + E = Union{Missing,T} + values = Union{Missing,Vector{E}}[ + missing, + E[], + E[missing, T((Int32(1), missing)), T((Int32(2), "b"))], + ] + input = (rows=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == + ["schema", "rows", "list", "element", "id", "label"] + @test result.metadata.schema[4].repetition_type == + WNMD.FieldRepetitionType.OPTIONAL + @test result.metadata.schema[4].num_children == 2 + @test result.metadata.schema[6].logicalType.STRING !== nothing + @test [leaf.path for leaf in result.schema.leaves] == [ + ["rows", "list", "element", "id"], + ["rows", "list", "element", "label"], + ] + wnexpectstream(result.streams[1], [0,0,0,1,1], + [0,1,2,3,3], Int32[1,2]; + max_repetition=1, max_definition=3) + wnexpectstream(result.streams[2], [0,0,0,1,1], + [0,1,2,3,4], wnbytes(["b"]); + max_repetition=1, max_definition=4) + table = Parquet.Table(bytes) + try + column = table.columns.rows + @test column[1] === missing + @test isempty(column[2]) + @test column[3][1] === missing + @test column[3][2]["id"] == 1 + @test column[3][2]["label"] === missing + @test column[3][3]["label"] == "b" + finally + close(table) + end + end +end + +@testset "recursive writer struct with LIST" begin + E = Union{Missing,Int32} + R = NamedTuple{ + (:id,:items), + Tuple{Int32,Union{Missing,Vector{E}}}, + } + values = Union{Missing,R}[ + missing, + R((Int32(1), missing)), + R((Int32(2), E[])), + R((Int32(3), E[missing,7])), + ] + input = (record=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == [ + "schema", "record", "id", "items", "list", "element", + ] + @test result.metadata.schema[4].logicalType.LIST !== nothing + @test [leaf.path for leaf in result.schema.leaves] == [ + ["record", "id"], + ["record", "items", "list", "element"], + ] + wnexpectstream(result.streams[1], [0,0,0,0], [0,1,1,1], + Int32[1,2,3]; max_repetition=0, max_definition=1) + wnexpectstream(result.streams[2], [0,0,0,0,1], + [0,1,2,3,4], Int32[7]; + max_repetition=1, max_definition=4) + table = Parquet.Table(bytes) + try + column = table.columns.record + @test column[1] === missing + @test column[2]["items"] === missing + @test isempty(column[3]["items"]) + @test isequal(collect(column[4]["items"]), E[missing,7]) + finally + close(table) + end + end +end + +@testset "recursive writer LIST of MAP" begin + maps = Parquet.MapVector( + Int32[0,0,2,3], + String["a","b","c"], + Union{Missing,Int32}[missing,2,3], + ) + values = Parquet.ListVector( + Int32[0,0,0,2,3], maps; + validity=Bool[false,true,true,true], + ) + input = (batches=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == [ + "schema", "batches", "list", "element", "key_value", "key", "value", + ] + @test result.metadata.schema[2].logicalType.LIST !== nothing + @test result.metadata.schema[4].logicalType.MAP !== nothing + @test [leaf.path for leaf in result.schema.leaves] == [ + ["batches", "list", "element", "key_value", "key"], + ["batches", "list", "element", "key_value", "value"], + ] + wnexpectstream(result.streams[1], [0,0,0,1,2,0], + [0,1,2,3,3,3], wnbytes(["a","b","c"]); + max_repetition=2, max_definition=3) + wnexpectstream(result.streams[2], [0,0,0,1,2,0], + [0,1,2,3,4,4], Int32[2,3]; + max_repetition=2, max_definition=4) + table = Parquet.Table(bytes) + try + column = table.columns.batches + @test column[1] === missing + @test isempty(column[2]) + @test isempty(column[3][1]) + @test isequal(collect(column[3][2]), + Pair{String,Union{Missing,Int32}}["a" => missing, "b" => 2]) + @test collect(column[4][1]) == ["c" => Int32(3)] + finally + close(table) + end + end +end + +@testset "recursive writer MAP of optional structs" begin + structs = Parquet.StructVector( + ["x","y"], + (Int32[1,2], Union{Missing,String}[missing,"z"]); + ranks=Int32[0,0,1,2], + ) + values = Parquet.MapVector( + Int32[0,0,0,3], String["a","b","c"], structs; + validity=Bool[false,true,true], + ) + input = (objects=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == [ + "schema", "objects", "key_value", "key", "value", "x", "y", + ] + @test result.metadata.schema[5].repetition_type == + WNMD.FieldRepetitionType.OPTIONAL + @test result.metadata.schema[5].num_children == 2 + wnexpectstream(result.streams[1], [0,0,0,1,1], + [0,1,2,2,2], wnbytes(["a","b","c"]); + max_repetition=1, max_definition=2) + wnexpectstream(result.streams[2], [0,0,0,1,1], + [0,1,2,3,3], Int32[1,2]; + max_repetition=1, max_definition=3) + wnexpectstream(result.streams[3], [0,0,0,1,1], + [0,1,2,3,4], wnbytes(["z"]); + max_repetition=1, max_definition=4) + table = Parquet.Table(bytes) + try + column = table.columns.objects + @test column[1] === missing + @test isempty(column[2]) + @test column[3][1].second === missing + @test column[3][2].second["x"] == 1 + @test column[3][2].second["y"] === missing + @test column[3][3].second["y"] == "z" + finally + close(table) + end + end +end + +@testset "recursive writer nested LIST" begin + Scalar = Union{Missing,Int32} + Inner = Vector{Scalar} + OuterElement = Union{Missing,Inner} + Outer = Vector{OuterElement} + values = Union{Missing,Outer}[ + missing, + OuterElement[], + OuterElement[ + missing, + Scalar[], + Scalar[missing,1,2], + Scalar[3], + ], + ] + input = (nested=values,) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pageversion=pageversion) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == [ + "schema", "nested", "list", "element", "list", "element", + ] + @test result.metadata.schema[2].logicalType.LIST !== nothing + @test result.metadata.schema[4].logicalType.LIST !== nothing + wnexpectstream(only(result.streams), [0,0,0,1,1,2,2,1], + [0,1,2,3,4,5,5,5], Int32[1,2,3]; + max_repetition=2, max_definition=5) + wncheckpagetype(bytes, 1, pageversion, 8, 3, 5) + table = Parquet.Table(bytes) + try + column = table.columns.nested + @test column[1] === missing + @test isempty(column[2]) + @test column[3][1] === missing + @test isempty(column[3][2]) + @test isequal(collect(column[3][3]), Scalar[missing,1,2]) + @test collect(column[3][4]) == Int32[3] + finally + close(table) + end + end +end + +@testset "recursive writer empty, all-null, and zero-row states" begin + S = NamedTuple{(:a,:b),Tuple{Int32,Union{Missing,String}}} + E = Union{Missing,Int32} + zero = ( + s=Union{Missing,S}[], + items=Union{Missing,Vector{E}}[], + attrs=Union{Missing,Dict{String,Union{Missing,Int32}}}[], + ) + for pageversion in (:v1, :v2) + result = wninspect(Parquet._encodefile(zero; + pageversion=pageversion)) + @test result.metadata.num_rows == 0 + @test isempty(result.metadata.row_groups) + @test isempty(result.streams) + @test result.metadata.schema[1].num_children == 3 + @test [element.name for element in result.metadata.schema] == [ + "schema", "s", "a", "b", "items", "list", "element", + "attrs", "key_value", "key", "value", + ] + end + + nullstructs = Union{Missing,S}[missing,missing,missing] + result = wninspect(Parquet._encodefile((s=nullstructs,))) + for stream in result.streams + @test stream.repetition == UInt64[0,0,0] + @test stream.definition == UInt64[0,0,0] + @test isempty(stream.values) + end + + Present = NamedTuple{ + (:a,:b), + Tuple{Union{Missing,Int32},Union{Missing,String}}, + } + present = Present[Present((missing,missing)), Present((missing,missing))] + result = wninspect(Parquet._encodefile((s=present,))) + for (index, stream) in enumerate(result.streams) + @test stream.repetition == UInt64[0,0] + @test stream.definition == UInt64[0,0] + @test result.schema.leaves[index].max_definition_level == 1 + @test isempty(stream.values) + end + + emptylists = Vector{Int32}[Int32[],Int32[]] + result = wninspect(Parquet._encodefile((items=emptylists,))) + wnexpectstream(only(result.streams), [0,0], [0,0], Int32[]; + max_repetition=1, max_definition=1) + + emptymaps = Dict{String,Int32}[Dict{String,Int32}(), Dict{String,Int32}()] + result = wninspect(Parquet._encodefile((attrs=emptymaps,))) + for stream in result.streams + wnexpectstream(stream, [0,0], [0,0], eltype(stream.values)[]; + max_repetition=1, max_definition=1) + end +end + +@testset "recursive writer duplicate struct names and selectors" begin + values = Parquet.StructVector( + ["x","x"], (Int32[1,2], String["a","b"])) + input = (s=values,) + bytes = Parquet._encodefile(input) + result = wninspect(bytes) + @test [element.name for element in result.metadata.schema] == + ["schema", "s", "x", "x"] + @test [leaf.path for leaf in result.schema.leaves] == + [["s", "x"], ["s", "x"]] + wnexpectstream(result.streams[1], [0,0], [0,0], Int32[1,2]; + max_repetition=0, max_definition=0) + wnexpectstream(result.streams[2], [0,0], [0,0], wnbytes(["a","b"]); + max_repetition=0, max_definition=0) + table = Parquet.Table(bytes) + try + @test table.columns.s[1][1] == 1 + @test table.columns.s[1][2] == "a" + @test_throws ArgumentError table.columns.s[1]["x"] + finally + close(table) + end + + @test_throws ArgumentError Parquet._encodefile(input; + encoding=((:s,:x) => :plain)) + selected = Parquet._encodefile(input; encoding=Dict{Any,Any}( + 1 => :delta_binary_packed, + 2 => :delta_byte_array, + )) + selectedmetadata = wninspect(selected).metadata + @test WNMD.Encoding.DELTA_BINARY_PACKED in + selectedmetadata.row_groups[1].columns[1].meta_data.encodings + @test WNMD.Encoding.DELTA_BYTE_ARRAY in + selectedmetadata.row_groups[1].columns[2].meta_data.encodings +end + +@testset "recursive writer duplicate MAP keys" begin + values = Parquet.MapVector( + Int32[0,3], String["k","k","z"], Int32[1,2,3]) + bytes = Parquet._encodefile((m=values,)) + result = wninspect(bytes) + wnexpectstream(result.streams[1], [0,1,1], [1,1,1], + wnbytes(["k","k","z"]); max_repetition=1, + max_definition=1) + wnexpectstream(result.streams[2], [0,1,1], [1,1,1], + Int32[1,2,3]; max_repetition=1, max_definition=1) + table = Parquet.Table(bytes) + try + value = table.columns.m[1] + @test collect(value) == ["k" => Int32(1), "k" => Int32(2), + "z" => Int32(3)] + @test Parquet.maplookup(value, "k") == 2 + @test Dict(value)["k"] == 2 + finally + close(table) + end +end + +function wnlogicalpayload(values) + return Parquet.StructVector( + ["enum","time","timestamp","decimal"], values) +end + +function wnchecklogicalschema(metadata::WNMD.FileMetaData) + @test [element.name for element in metadata.schema] == [ + "schema", "logicals", "list", "element", "enum", "time", + "timestamp", "decimal", + ] + enum, time, timestamp, decimal = metadata.schema[5:8] + @test enum.type_ == WNMD.Type.BYTE_ARRAY + @test enum.logicalType.ENUM !== nothing + @test enum.converted_type == WNMD.ConvertedType.ENUM + @test time.type_ == WNMD.Type.INT64 + @test time.logicalType.TIME.unit.MICROS !== nothing + @test time.logicalType.TIME.isAdjustedToUTC + @test time.converted_type == WNMD.ConvertedType.TIME_MICROS + @test timestamp.type_ == WNMD.Type.INT64 + @test timestamp.logicalType.TIMESTAMP.unit.NANOS !== nothing + @test !timestamp.logicalType.TIMESTAMP.isAdjustedToUTC + @test timestamp.converted_type === nothing + @test decimal.type_ == WNMD.Type.FIXED_LEN_BYTE_ARRAY + @test decimal.type_length == 9 + @test decimal.precision == 20 + @test decimal.scale == 4 + @test decimal.logicalType.DECIMAL.precision == 20 + @test decimal.logicalType.DECIMAL.scale == 4 + @test decimal.converted_type == WNMD.ConvertedType.DECIMAL + return +end + +@testset "nested local millisecond timestamp compatibility annotation" begin + input = (events=[(at=DateTime(1970, 1, 1),)],) + metadata = wninspect(Parquet._encodefile(input)).metadata + timestamp = only(filter(element -> element.name == "at", metadata.schema)) + @test timestamp.logicalType.TIMESTAMP.unit.MILLIS !== nothing + @test !timestamp.logicalType.TIMESTAMP.isAdjustedToUTC + @test timestamp.converted_type == WNMD.ConvertedType.TIMESTAMP_MILLIS +end + +@testset "recursive writer explicit logical leaves" begin + missingvalues = ( + Parquet.LogicalColumn(Missing[missing,missing], :enum), + Parquet.LogicalColumn(Missing[missing,missing], :time; + unit=:micros, adjusted=true), + Parquet.LogicalColumn(Missing[missing,missing], :timestamp; + unit=:nanos, adjusted=false), + Parquet.LogicalColumn(Missing[missing,missing], :decimal; + precision=20, scale=4), + ) + input = (logicals=Parquet.ListVector( + Int32[0,0,2], wnlogicalpayload(missingvalues)),) + for pageversion in (:v1, :v2) + result = wninspect(Parquet._encodefile(input; + pageversion=pageversion)) + wnchecklogicalschema(result.metadata) + for stream in result.streams + wnexpectstream(stream, [0,0,1], [0,1,1], + eltype(stream.values)[]; max_repetition=1, + max_definition=2) + end + end + + presentvalues = ( + Parquet.LogicalColumn(Union{Missing,String}["alpha"], :enum), + Parquet.LogicalColumn(Union{Missing,Time}[Time(0)], :time; + unit=:micros, adjusted=true), + Parquet.LogicalColumn( + Union{Missing,Parquet.Timestamp{:nanos}}[ + Parquet.Timestamp(7, :nanos, false), + ], :timestamp; unit=:nanos, adjusted=false), + Parquet.LogicalColumn(Union{Missing,Parquet.Decimal}[ + Parquet.Decimal(12345, 4), + ], :decimal; precision=20, scale=4), + ) + present = (logicals=Parquet.ListVector( + Int32[0,1], wnlogicalpayload(presentvalues)),) + result = wninspect(Parquet._encodefile(present)) + wnchecklogicalschema(result.metadata) + for stream in result.streams + @test stream.repetition == UInt64[0] + @test stream.definition == UInt64[2] + end + @test result.streams[1].values == wnbytes(["alpha"]) + @test result.streams[2].values == Int64[0] + @test result.streams[3].values == Int64[7] + @test result.streams[4].values == [ + UInt8[0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x30,0x39], + ] + table = Parquet.Table(Parquet._encodefile(present)) + try + value = table.columns.logicals[1][1] + @test value["enum"] == "alpha" + @test value["time"] == Time(0) + @test value["timestamp"] == Parquet.Timestamp(7, :nanos, false) + @test value["decimal"] == Parquet.Decimal(12345, 4) + finally + close(table) + end +end + +@testset "recursive writer nested codec and encoding matrix" begin + C = NamedTuple{ + (:flag,:i32,:i64,:f32,:f64,:text,:fixed), + Tuple{Bool,Int32,Int64,Float32,Float64,String,NTuple{2,UInt8}}, + } + entries = C[ + C((isodd(index), Int32(index % 7), Int64(index % 11), + Float32(index % 5), Float64(index % 13), + "value-$(index % 4)", (UInt8(index % 4), UInt8(0x7f)))) + for index in 1:128 + ] + input = (records=Vector{C}[entries],) + codecs = ( + :uncompressed, + :snappy, + :gzip, + :brotli, + :zstd, + :lz4_raw, + ) + for pageversion in (:v1, :v2), codec in codecs + bytes = Parquet._encodefile(input; pageversion=pageversion, + codec=codec) + table = Parquet.Table(bytes) + try + records = table.columns.records[1] + @test length(records) == length(entries) + @test records[1]["i32"] == entries[1].i32 + @test records[64]["text"] == entries[64].text + @test records[end]["fixed"] == collect(entries[end].fixed) + finally + close(table) + end + metadata = wninspect(bytes).metadata + expectedcodec = Parquet._writecodec(codec) + @test all(chunk.meta_data.codec == expectedcodec for chunk in + only(metadata.row_groups).columns) + end + + paths = Dict{Any,Any}( + (:records,:list,:element,:flag) => :rle, + (:records,:list,:element,:i32) => :delta_binary_packed, + (:records,:list,:element,:i64) => :delta_binary_packed, + (:records,:list,:element,:f32) => :byte_stream_split, + (:records,:list,:element,:f64) => :byte_stream_split, + (:records,:list,:element,:text) => :delta_byte_array, + (:records,:list,:element,:fixed) => :delta_byte_array, + ) + encodings = ( + WNMD.Encoding.RLE, + WNMD.Encoding.DELTA_BINARY_PACKED, + WNMD.Encoding.DELTA_BINARY_PACKED, + WNMD.Encoding.BYTE_STREAM_SPLIT, + WNMD.Encoding.BYTE_STREAM_SPLIT, + WNMD.Encoding.DELTA_BYTE_ARRAY, + WNMD.Encoding.DELTA_BYTE_ARRAY, + ) + for pageversion in (:v1, :v2) + metadata = wninspect(Parquet._encodefile(input; + pageversion=pageversion, encoding=paths)).metadata + for (chunk, encoding) in zip(only(metadata.row_groups).columns, + encodings) + @test encoding in chunk.meta_data.encodings + end + dictionary = wninspect(Parquet._encodefile(input; + pageversion=pageversion, + encoding=((:records,:list,:element,:text) => :dictionary), + )).metadata + textchunk = only(dictionary.row_groups).columns[6].meta_data + @test textchunk.dictionary_page_offset !== nothing + @test WNMD.Encoding.RLE_DICTIONARY in textchunk.encodings + end +end + +@testset "recursive writer malformed inputs" begin + Empty = NamedTuple{(),Tuple{}} + @test_throws ArgumentError Parquet._encodefile((value=Empty[Empty(())],)) + @test_throws ArgumentError Parquet._encodefile((value= + Parquet.StructVector(String[], (); rows=1),)) + + Required = NamedTuple{(:x,),Tuple{Int32}} + required = WNDeclaredVector{Required}(Any[missing]) + @test_throws ArgumentError Parquet._encodefile((value=required,)) + + badmaps = WNBadMap[WNBadMap()] + @test_throws ArgumentError Parquet._encodefile((value=badmaps,)) + nullablekeys = Dict{Union{Missing,String},Int32}[ + Dict{Union{Missing,String},Int32}("x" => Int32(1)), + ] + @test_throws ArgumentError Parquet._encodefile((value=nullablekeys,)) + + @test_throws ArgumentError Parquet._encodefile((value=Any[Int32[1]],)) + abstractlists = AbstractVector{Int32}[Int32[1]] + @test_throws ArgumentError Parquet._encodefile((value=abstractlists,)) + heterogeneous = Union{Vector{Int32},Vector{String}}[Int32[1], String["x"]] + @test_throws ArgumentError Parquet._encodefile((value=heterogeneous,)) + unparameterized = Vector[Int32[1]] + @test_throws ArgumentError Parquet._encodefile((value=unparameterized,)) + + @test_throws ArgumentError Parquet._encodefile( + (items=Vector{Int32}[Int32[1]],); + encoding=((:items,:list,:element) => :delta_byte_array)) + @test_throws ArgumentError Parquet._encodefile( + (items=Vector{Int32}[Int32[1]],); + encoding=((:missing,:list,:element) => :plain)) +end + +@testset "recursive writer resource preflight and arbitrary axes" begin + for limits in ( + Parquet.Limits(max_container_elements=1), + Parquet.Limits(max_materialized_bytes=1024), + ) + source = WNTrackedRows{Base.HasLength}(Ref(0), 100_000) + budget = Parquet._LiveByteBudget(limits) + @test_throws Parquet.LimitError Parquet._writefields(source, + limits, budget) + @test source.consumed[] == 0 + @test Parquet._budgetused(budget) == 0 + end + + unknown = WNTrackedRows{Base.SizeUnknown}(Ref(0), 100_000) + unknownlimits = Parquet.Limits(max_container_elements=1) + unknownbudget = Parquet._LiveByteBudget(unknownlimits) + @test_throws Parquet.LimitError Parquet._writefields(unknown, + unknownlimits, unknownbudget) + @test unknown.consumed[] == 2 + @test Parquet._budgetused(unknownbudget) == 0 + + schemaless = WNSchemaLessRows([ + (value=Int32(1), items=Int32[1,2]), + (value=Int32(2), items=Int32[]), + ]) + table = Parquet.Table(Parquet._encodefile(schemaless)) + try + @test table.columns.value == Int32[1,2] + @test collect(table.columns.items[1]) == Int32[1,2] + @test isempty(table.columns.items[2]) + finally + close(table) + end + unnamed = WNSchemaLessRows(NTuple{3,Int32}[]) + unnamedlimits = Parquet.Limits(max_materialized_bytes=1, + max_container_elements=1) + unnamedbudget = Parquet._LiveByteBudget(unnamedlimits) + @test_throws ArgumentError Parquet._writefields(unnamed, + unnamedlimits, unnamedbudget) + @test Parquet._budgetused(unnamedbudget) == 0 + + wide = WNSchemaLessRows( + NamedTuple{(:left,:right),Tuple{Int32,Int32}}[]) + widebudget = Parquet._LiveByteBudget(unnamedlimits) + @test_throws Parquet.LimitError Parquet._writefields(wide, + unnamedlimits, widebudget) + @test Parquet._budgetused(widebudget) == 0 + + for size in (Base.HasLength, Base.SizeUnknown) + fresh = WNFreshStringRows{size}(Ref(0), 20, 100_000) + freshlimits = Parquet.Limits(max_materialized_bytes=100_000) + freshbudget = Parquet._LiveByteBudget(freshlimits) + @test_throws Parquet.LimitError Parquet._writefields(fresh, + freshlimits, freshbudget) + @test fresh.consumed[] == 1 + @test Parquet._budgetused(freshbudget) == 0 + end + + for (type, maker) in ( + (Parquet.JSONValue, wnfreshjson), + (Parquet.BSONValue, wnfreshbson), + (Parquet.Decimal, wnfreshdecimal), + ) + fresh = WNFreshPayloadRows{type,typeof(maker)}( + Ref(0), 20, 100_000, maker) + freshlimits = Parquet.Limits(max_materialized_bytes=200_000) + freshbudget = Parquet._LiveByteBudget(freshlimits) + @test_throws Parquet.LimitError Parquet._writefields(fresh, + freshlimits, freshbudget) + @test fresh.consumed[] == 2 + @test Parquet._budgetused(freshbudget) == 0 + end + + UnionType = Union{Int32,Float32} + @test Parquet._materializedarraybytes(UnionType, 4; header=false) == + 4 * (Base.elsize(Vector{UnionType}) + 1) + + tracked = WNTrackedBytes(fill(UInt8(1), 100_000), Ref(0)) + pagelimits = Parquet.Limits(max_page_bytes=1) + payloadbudget = Parquet._LiveByteBudget(pagelimits) + @test_throws Parquet.LimitError Parquet._writefields( + (value=WNTrackedBytes[tracked],), pagelimits, payloadbudget) + @test tracked.reads[] == 0 + @test Parquet._budgetused(payloadbudget) == 0 + + large = repeat("y", 100_000) + wnmutationattack(large) + @test @allocated(wnmutationattack(large)) < 50_000 + + scalar = WNZeroVector(Int32[10,20]) + table = Parquet.Table(Parquet._encodefile((value=scalar,))) + try + @test table.columns.value == Int32[10,20] + finally + close(table) + end + + nested = WNZeroVector([ + WNZeroVector(Int32[1,2]), + WNZeroVector(Int32[]), + ]) + table = Parquet.Table(Parquet._encodefile((value=nested,))) + try + @test collect(table.columns.value[1]) == Int32[1,2] + @test isempty(table.columns.value[2]) + finally + close(table) + end +end + +@testset "ordinary writer iterative topology and exact rollback" begin + depth = 4096 + exactlimits = Parquet.Limits(max_metadata_depth=depth + 2, + max_container_elements=100_000, + max_materialized_bytes=256 * 1024 * 1024) + sources = (wndeepstruct(depth, 0), wndeepstruct(depth, 1)) + for (rows, source) in enumerate(sources) + expectedrows = rows - 1 + budget = Parquet._LiveByteBudget(exactlimits) + fields, written = Parquet._writefields((deep=source,), exactlimits, + budget) + field = only(fields) + leaf = only(field.leaves) + @test written == expectedrows + @test length(field.schema) == depth + 1 + @test length(leaf.path) == depth + 1 + @test leaf.values == (iszero(expectedrows) ? Int32[] : Int32[7]) + end + + precharge = Int64(257) + failurelimits = Parquet.Limits(max_metadata_depth=depth + 1, + max_container_elements=100_000, + max_materialized_bytes=256 * 1024 * 1024) + failurebudget = Parquet._LiveByteBudget(failurelimits) + Parquet._reserve!(failurebudget, precharge) + caught = try + Parquet._writefields((deep=last(sources),), failurelimits, + failurebudget) + nothing + catch err + err + end + @test caught isa Parquet.LimitError + @test caught.resource == :metadata_depth + @test caught.requested == depth + 2 + @test caught.maximum == depth + 1 + @test Parquet._budgetused(failurebudget) == precharge + Parquet._release!(failurebudget, precharge) + + frame = Parquet._NestedWriteSourceFrame + headercharge = Parquet._materializedsum( + Parquet._materializedarraybytes(frame, 0), + Parquet._MATERIALIZED_OBJECT_BYTES) + framecharge = Parquet._materializedarraybytes(frame, 1; header=false) + traversalcharge = headercharge + 2 * framecharge + framesource = Parquet.StructVector(String["value"], + AbstractVector[Int32[]]; rows=0) + lowlimits = Parquet.Limits( + max_materialized_bytes=precharge + traversalcharge - 1) + lowbudget = Parquet._LiveByteBudget(lowlimits) + Parquet._reserve!(lowbudget, precharge) + caught = try + Parquet._nestedwritevalidatesource(framesource, lowlimits, lowbudget) + nothing + catch err + err + end + @test caught isa Parquet.LimitError + @test caught.resource == :materialized_bytes + @test caught.requested == precharge + traversalcharge + @test caught.maximum == precharge + traversalcharge - 1 + @test Parquet._budgetused(lowbudget) == precharge + Parquet._release!(lowbudget, precharge) + + exactframelimits = Parquet.Limits( + max_materialized_bytes=precharge + traversalcharge) + exactframebudget = Parquet._LiveByteBudget(exactframelimits) + Parquet._reserve!(exactframebudget, precharge) + Parquet._nestedwritevalidatesource(framesource, exactframelimits, + exactframebudget) + @test Parquet._budgetused(exactframebudget) == precharge + Parquet._release!(exactframebudget, precharge) + + precedencecharge = headercharge + framecharge + precedencesource = Parquet.StructVector(String["first", "second"], + AbstractVector[Int32[], Int32[]]; rows=0) + precedencelimits = Parquet.Limits(max_metadata_depth=1, + max_materialized_bytes=precharge + precedencecharge) + precedencebudget = Parquet._LiveByteBudget(precedencelimits) + Parquet._reserve!(precedencebudget, precharge) + caught = try + Parquet._nestedwritevalidatesource(precedencesource, + precedencelimits, precedencebudget) + nothing + catch err + err + end + @test caught isa Parquet.LimitError + @test caught.resource == :metadata_depth + @test caught.requested == 2 + @test caught.maximum == 1 + @test Parquet._budgetused(precedencebudget) == precharge + Parquet._release!(precedencebudget, precharge) + + sentinel = ErrorException("writer topology callback sentinel") + backing = WNThrowingSizeVector(Int32[7], 0, 4, sentinel) + view = Parquet.ListValue(backing, 1, 1) + views = WNDeclaredVector{Parquet.ListValue{Int32}}(Any[view]) + callbacklimits = Parquet.Limits() + callbackbudget = Parquet._LiveByteBudget(callbacklimits) + Parquet._reserve!(callbackbudget, precharge) + caught = try + Parquet._writefields((deep=views,), callbacklimits, callbackbudget) + nothing + catch err + err + end + @test caught === sentinel + @test backing.calls == 4 + @test Parquet._budgetused(callbackbudget) == precharge + Parquet._release!(callbackbudget, precharge) +end + +@testset "ordinary writer reuses pass scratch stacks" begin + rows = 1000 + structs = Parquet.StructVector(String["value"], + AbstractVector[fill(Int32(1), rows)]; rows=rows) + lists = Parquet.ListVector(collect(Int32(0):Int32(rows)), + fill(Int32(1), rows)) + structscan, structrows = wnwriterpassallocations(structs) + listscan, listrows = wnwriterpassallocations(lists) + # These bounds catch a pass that stops reusing its scratch stacks, which costs + # far more than a constant per row. Keep them loose: the same measurement runs + # about a seventh higher on x86-64 than on arm64, and varies by Julia version. + @test structscan < 1800 * rows + @test structrows < 3300 * rows + @test listscan < 4200 * rows + @test listrows < 6300 * rows + + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + shape = Parquet._nestedwriteshape("value", Int32, Int32[], limits, + budget) + fragment = Parquet._nestedwriteschema(shape, limits, budget) + semantic, plans = Parquet._nestedwritecompile( + Parquet._NestedWriteShape[shape], + Vector{Parquet.Metadata.SchemaElement}[fragment], limits, budget) + counts = Parquet._nestedwritecounts(length(semantic.leaves), budget) + context = Parquet._NestedWriteCountContext(counts, limits) + plan = only(plans)::Parquet._NestedWriteLeafPlan + Parquet._nestedwritescanaggregate!(shape, Int32(1), limits) + Parquet._nestedwriteshred!(context, plan, Int32(1), UInt64(0), nothing) + @test @allocated(Parquet._nestedwritescanaggregate!(shape, Int32(1), + limits)) == 0 + @test @allocated(Parquet._nestedwriteshred!(context, plan, Int32(1), + UInt64(0), nothing)) == 0 + + precharge = Int64(31) + actiontype = Parquet._NestedWriteScanAction + headercharge = Parquet._materializedsum( + Parquet._materializedarraybytes(actiontype, 0), + Parquet._MATERIALIZED_OBJECT_BYTES) + framecharge = Parquet._materializedarraybytes(actiontype, 1; + header=false) + lowlimits = Parquet.Limits( + max_materialized_bytes=precharge + headercharge + framecharge - 1) + lowbudget = Parquet._LiveByteBudget(lowlimits) + Parquet._reserve!(lowbudget, precharge) + lowstack = Parquet._nestedwritepassstackstart(actiontype, lowbudget) + action = Parquet._nestedwritescanenter(shape, Int32(1), nothing) + caught = try + Parquet._nestedwritestackpush!(lowstack, action, lowbudget) + nothing + catch err + err + end + @test caught isa Parquet.LimitError + @test caught.requested == precharge + headercharge + framecharge + @test Parquet._budgetused(lowbudget) == precharge + headercharge + Parquet._nestedwritepassstackrelease!(lowstack, lowbudget) + @test Parquet._budgetused(lowbudget) == precharge + Parquet._release!(lowbudget, precharge) + + exactlimits = Parquet.Limits( + max_materialized_bytes=precharge + headercharge + framecharge) + exactbudget = Parquet._LiveByteBudget(exactlimits) + Parquet._reserve!(exactbudget, precharge) + exactstack = Parquet._nestedwritepassstackstart(actiontype, exactbudget) + Parquet._nestedwritestackpush!(exactstack, action, exactbudget) + Parquet._nestedwritepassstackpop!(exactstack) + Parquet._nestedwritepassstackprocessed!(exactstack) + @test Parquet._budgetused(exactbudget) == + precharge + headercharge + framecharge + Parquet._nestedwritepassstackrelease!(exactstack, exactbudget) + @test Parquet._budgetused(exactbudget) == precharge + Parquet._release!(exactbudget, precharge) +end + +@testset "aggregate scan without a trace covers MAP sources" begin + # A MAP key snapshot exists only while a trace records one, so a traceless + # aggregate scan has no snapshot to assert against. Both MAP scan arms used to + # reach the key assertion with no snapshot and raise a MethodError, so this + # helper was only ever exercised with struct and list sources. + rows = 4 + dicts = [Dict("k$(index)" => Int32(index)) for index in 1:rows] + dictscan, dictrows = wnwriterpassallocations(dicts) + @test dictscan >= 0 && dictrows >= 0 + + views = Parquet.MapVector(collect(Int32(0):Int32(rows)), + String["k$(index)" for index in 1:rows], collect(Int32(1):Int32(rows))) + viewscan, viewrows = wnwriterpassallocations(views) + @test viewscan >= 0 && viewrows >= 0 + + # Inferring the leaf schema is the reason a traceless scan exists, so check that + # it still aggregates rather than merely completing. + limits = Parquet.Limits(max_materialized_bytes=1_000_000_000) + function wnmapaggregate(source) + budget = Parquet._LiveByteBudget(limits) + shape = Parquet._nestedwriteshape("value", eltype(source), source, + limits, budget) + Parquet._nestedwritescanaggregates!(Parquet._NestedWriteShape[shape], + AbstractVector[source], length(source), limits, nothing) + return shape.value.aggregate + end + decimals = Dict{String,Parquet.Decimal}[ + Dict("k" => Parquet.Decimal(Int128(1234), Int32(2)))] + decimal = wnmapaggregate(decimals) + @test decimal.seen && decimal.precision == 4 && decimal.scale == Int32(2) + stamps = Dict{String,Parquet.Timestamp{:micros}}[ + Dict("k" => Parquet.Timestamp(Int64(5), :micros, true))] + stamp = wnmapaggregate(stamps) + @test stamp.seen && something(stamp.adjusted) + + # The snapshot is the only check the traceless path drops; a key that does not + # match its declared shape must still be rejected. + budget = Parquet._LiveByteBudget(limits) + keyshape = Parquet._nestedwriteshape("key", String, String[], limits, budget) + @test_throws ArgumentError Parquet._nestedwritekeyassert!(nothing, Int32(1), + keyshape, limits, nothing, nothing, nothing) + @test Parquet._nestedwritekeyassert!(nothing, "k", keyshape, limits, nothing, + nothing, nothing) === nothing +end diff --git a/test/write_offset_index.jl b/test/write_offset_index.jl new file mode 100644 index 0000000..4aabb43 --- /dev/null +++ b/test/write_offset_index.jl @@ -0,0 +1,1423 @@ +using Dates +using Test + +const OIMD = Parquet.Metadata +const OITH = Parquet.Thrift + +function oireplace(value; replacements...) + names = fieldnames(typeof(value)) + fields = map(names) do name + return haskey(replacements, name) ? replacements[name] : + getfield(value, name) + end + return typeof(value)(fields...) +end + +function oimetadata(file::Parquet.File) + reader = OITH.Reader(file.footer.bytes) + metadata = OITH.decode(reader, OIMD.FileMetaData) + @test OITH.remaining(reader) == 0 + return metadata +end + +function oimetadata(bytes::AbstractVector{UInt8}) + file = Parquet.File(bytes) + try + return oimetadata(file) + finally + close(file) + end +end + +function oichunkstart(metadata::OIMD.ColumnMetaData) + offsets = Int64[] + metadata.data_page_offset > 0 && push!(offsets, + Int64(metadata.data_page_offset)) + dictionary = metadata.dictionary_page_offset + dictionary !== nothing && dictionary > 0 && push!(offsets, + Int64(dictionary)) + index = metadata.index_page_offset + index !== nothing && push!(offsets, Int64(index)) + isempty(offsets) && return Int64(metadata.data_page_offset) + return minimum(offsets) +end + +function oiframes(file::Parquet.File, chunk::OIMD.ColumnChunk) + metadata = something(chunk.meta_data) + start = oichunkstart(metadata) + stop = Base.checked_add(start, metadata.total_compressed_size) + output = [] + position = start + budget = Parquet._LiveByteBudget(Parquet.Limits()) + while position < stop + frame = Parquet.readpage(file.source, position, stop, + Parquet.Limits(); budget=budget) + frameend = Parquet.pageend(frame) + push!(output, ( + offset=position, + header=frame.header, + headerlength=frame.headerlength, + payload=collect(frame.payload), + frameend=frameend, + )) + Parquet._release!(budget, frame.materializedcharge) + position = frameend + end + @test position == stop + return output +end + +function oirawindex(bytes::AbstractVector{UInt8}, footer::Int64, + chunk::OIMD.ColumnChunk) + offset = chunk.offset_index_offset + count = chunk.offset_index_length + @test offset !== nothing + @test count !== nothing + offset = Int64(offset) + count = Int64(count) + @test offset >= 0 + @test count > 0 + @test Base.checked_add(offset, count) <= footer + first = Int(offset) + 1 + last = Int(offset) + Int(count) + raw = @view bytes[first:last] + @test length(raw) == count + reader = OITH.Reader(raw) + index = OITH.decode(reader, OIMD.OffsetIndex) + @test OITH.remaining(reader) == 0 + return index, collect(raw) +end + +function oidatapage(frame) + type = frame.header.type_ + return type == OIMD.PageType.DATA_PAGE || + type == OIMD.PageType.DATA_PAGE_V2 +end + +function oipagevalues(frame) + header = frame.header + header.type_ == OIMD.PageType.DATA_PAGE && + return Int64(header.data_page_header.num_values) + header.type_ == OIMD.PageType.DATA_PAGE_V2 && + return Int64(header.data_page_header_v2.num_values) + return Int64(0) +end + +function oipagerows(frame, metadata::OIMD.ColumnMetaData, + node::Parquet.SchemaNode) + header = frame.header + if header.type_ == OIMD.PageType.DATA_PAGE_V2 + page = header.data_page_header_v2 + count = Int(page.num_values) + node.max_repetition_level == 0 && return Int64(page.num_rows) + bytes = @view frame.payload[1:Int(page.repetition_levels_byte_length)] + repetition = Parquet._decodelevelsv2(bytes, count, + Int(node.max_repetition_level), "repetition", Parquet.Limits()) + return Int64(Base.count(iszero, repetition)) + end + page = header.data_page_header + count = Int(page.num_values) + node.max_repetition_level == 0 && return Int64(count) + payload = Parquet.decompress(metadata.codec, frame.payload, + header.uncompressed_page_size) + repetition, _ = Parquet._decodelevelv1(payload, count, + page.repetition_level_encoding, Int(node.max_repetition_level), 1, + "repetition", Parquet.Limits()) + return Int64(Base.count(iszero, repetition)) +end + +function oiexpectedlocations(frames, metadata::OIMD.ColumnMetaData, + node::Parquet.SchemaNode) + output = OIMD.PageLocation[] + row = Int64(0) + for frame in frames + oidatapage(frame) || continue + size = frame.frameend - frame.offset + @test 0 < size <= typemax(Int32) + push!(output, OIMD.PageLocation(offset=frame.offset, + compressed_page_size=Int32(size), first_row_index=row)) + row = Base.checked_add(row, oipagerows(frame, metadata, node)) + end + return output, row +end + +function oiinspect(bytes::Vector{UInt8}; pageindex::Bool=true) + file = Parquet.File(bytes) + try + metadata = oimetadata(file) + schema = Parquet.Schema(metadata) + groups = [] + indexgroups = [] + physicalintervals = Tuple{Int64,Int64}[] + indexintervals = Tuple{Int64,Int64}[] + for group in metadata.row_groups + frames = [] + indexes = Union{Nothing,OIMD.OffsetIndex}[] + compressed = Int64(0) + uncompressed = Int64(0) + for (chunk, node) in zip(group.columns, schema.leaves) + column = something(chunk.meta_data) + chunkframes = oiframes(file, chunk) + push!(frames, chunkframes) + start = oichunkstart(column) + stop = Base.checked_add(start, column.total_compressed_size) + push!(physicalintervals, (start, stop)) + framecompressed = sum(frame -> frame.frameend - frame.offset, + chunkframes; init=Int64(0)) + frameuncompressed = sum(frame -> Int64(frame.headerlength) + + Int64(frame.header.uncompressed_page_size), chunkframes; + init=Int64(0)) + @test framecompressed == column.total_compressed_size + @test frameuncompressed == column.total_uncompressed_size + @test sum(oipagevalues, chunkframes; init=Int64(0)) == + column.num_values + compressed = Base.checked_add(compressed, framecompressed) + uncompressed = Base.checked_add(uncompressed, + frameuncompressed) + dataframes = filter(oidatapage, chunkframes) + @test !isempty(dataframes) + @test column.data_page_offset == first(dataframes).offset + dictionaries = filter(frame -> + frame.header.type_ == OIMD.PageType.DICTIONARY_PAGE, + chunkframes) + if isempty(dictionaries) + @test column.dictionary_page_offset === nothing + else + @test length(dictionaries) == 1 + @test column.dictionary_page_offset == + only(dictionaries).offset + end + @test column.index_page_offset === nothing + if pageindex + index, raw = oirawindex(bytes, file.footer.offset, chunk) + expected, rows = oiexpectedlocations(chunkframes, column, + node) + @test index.page_locations == expected + @test index.unencoded_byte_array_data_bytes === nothing + @test rows == group.num_rows + @test length(raw) == chunk.offset_index_length + push!(indexes, index) + push!(indexintervals, (Int64(chunk.offset_index_offset), + Int64(chunk.offset_index_offset) + + Int64(chunk.offset_index_length))) + else + @test chunk.offset_index_offset === nothing + @test chunk.offset_index_length === nothing + push!(indexes, nothing) + end + end + @test group.total_compressed_size == compressed + @test group.total_byte_size == uncompressed + push!(groups, frames) + push!(indexgroups, indexes) + end + if !isempty(physicalintervals) + @test first(first(physicalintervals)) == 4 + for index in 2:length(physicalintervals) + @test physicalintervals[index - 1][2] == + physicalintervals[index][1] + end + if pageindex + @test last(physicalintervals)[2] == first(indexintervals)[1] + for index in 2:length(indexintervals) + @test indexintervals[index - 1][2] == + indexintervals[index][1] + end + @test last(indexintervals)[2] == file.footer.offset + else + @test last(physicalintervals)[2] == file.footer.offset + @test isempty(indexintervals) + end + end + return (; metadata, schema, groups, indexgroups, + footer_offset=file.footer.offset, physicalintervals, + indexintervals) + finally + close(file) + end +end + +function oiroundtrip(bytes::Vector{UInt8}, expected::NamedTuple) + table = Parquet.Table(bytes) + try + for name in keys(expected) + @test isequal(getproperty(table.columns, name), + getproperty(expected, name)) + end + finally + close(table) + end + return +end + +function oigoldeninput() + E = Union{Missing,Date} + days = Union{Missing,Vector{E}}[ + missing, + E[], + E[missing], + E[Date(1970, 1, 1), missing, Date(1969, 12, 31)], + E[Date(2000, 2, 29)], + ] + return (; id=Int32[1, 2, 3, 4, 5], days) +end + +function oirewritefooter(bytes::Vector{UInt8}, metadata::OIMD.FileMetaData) + file = Parquet.File(bytes) + offset = try + file.footer.offset + finally + close(file) + end + output = copy(bytes[1:Int(offset)]) + footer = OITH.encode(metadata) + append!(output, footer) + Parquet._writelittle!(output, UInt32(length(footer))) + append!(output, Parquet.PARQUET_MAGIC) + return output +end + +function oimetadatawithchunk(metadata::OIMD.FileMetaData, groupindex::Int, + columnindex::Int, chunk::OIMD.ColumnChunk) + groups = copy(metadata.row_groups) + group = groups[groupindex] + columns = copy(group.columns) + columns[columnindex] = chunk + groups[groupindex] = oireplace(group; columns=columns) + return oireplace(metadata; row_groups=groups) +end + +function oichunkrewrite(bytes::Vector{UInt8}, groupindex::Int, + columnindex::Int; replacements...) + metadata = oimetadata(bytes) + chunk = metadata.row_groups[groupindex].columns[columnindex] + replaced = oireplace(chunk; replacements...) + return oirewritefooter(bytes, oimetadatawithchunk(metadata, groupindex, + columnindex, replaced)) +end + +function oiindexraws(bytes::Vector{UInt8}, metadata::OIMD.FileMetaData) + file = Parquet.File(bytes) + try + return [Any[chunk.offset_index_offset === nothing ? nothing : + last(oirawindex(bytes, file.footer.offset, chunk)) + for chunk in group.columns] for group in metadata.row_groups] + finally + close(file) + end +end + +function oiphysicalend(metadata::OIMD.FileMetaData) + stop = Int64(4) + for group in metadata.row_groups, chunk in group.columns + column = something(chunk.meta_data) + start = oichunkstart(column) + stop = max(stop, Base.checked_add(start, + column.total_compressed_size)) + end + return stop +end + +function oirebuildsections(bytes::Vector{UInt8}; indexes=nothing, + columns=nothing) + metadata = oimetadata(bytes) + indexraws = indexes === nothing ? oiindexraws(bytes, metadata) : indexes + columnraws = columns === nothing ? + [Any[nothing for _ in group.columns] + for group in metadata.row_groups] : columns + length(indexraws) == length(metadata.row_groups) || + throw(ArgumentError("offset-index group count differs")) + length(columnraws) == length(metadata.row_groups) || + throw(ArgumentError("column-index group count differs")) + bodyend = oiphysicalend(metadata) + output = copy(bytes[1:Int(bodyend)]) + chunks = [copy(group.columns) for group in metadata.row_groups] + for groupindex in eachindex(chunks) + length(columnraws[groupindex]) == length(chunks[groupindex]) || + throw(ArgumentError("column-index leaf count differs")) + for columnindex in eachindex(chunks[groupindex]) + raw = columnraws[groupindex][columnindex] + if raw === nothing + chunks[groupindex][columnindex] = oireplace( + chunks[groupindex][columnindex]; + column_index_offset=nothing, + column_index_length=nothing) + continue + end + encoded = raw isa AbstractVector{UInt8} ? collect(raw) : + OITH.encode(raw) + offset = Int64(length(output)) + append!(output, encoded) + chunks[groupindex][columnindex] = oireplace( + chunks[groupindex][columnindex]; + column_index_offset=offset, + column_index_length=Int32(length(encoded))) + end + end + for groupindex in eachindex(chunks) + length(indexraws[groupindex]) == length(chunks[groupindex]) || + throw(ArgumentError("offset-index leaf count differs")) + for columnindex in eachindex(chunks[groupindex]) + raw = indexraws[groupindex][columnindex] + if raw === nothing + chunks[groupindex][columnindex] = oireplace( + chunks[groupindex][columnindex]; + offset_index_offset=nothing, + offset_index_length=nothing) + continue + end + encoded = raw isa AbstractVector{UInt8} ? collect(raw) : + OITH.encode(raw) + offset = Int64(length(output)) + append!(output, encoded) + chunks[groupindex][columnindex] = oireplace( + chunks[groupindex][columnindex]; + offset_index_offset=offset, + offset_index_length=Int32(length(encoded))) + end + end + groups = OIMD.RowGroup[ + oireplace(group; columns=chunks[index]) + for (index, group) in enumerate(metadata.row_groups) + ] + rebuilt = oireplace(metadata; row_groups=groups) + footer = OITH.encode(rebuilt) + append!(output, footer) + Parquet._writelittle!(output, UInt32(length(footer))) + append!(output, Parquet.PARQUET_MAGIC) + return output +end + +function oiindexobjects(bytes::Vector{UInt8}) + metadata = oimetadata(bytes) + file = Parquet.File(bytes) + try + return [Any[chunk.offset_index_offset === nothing ? nothing : + first(oirawindex(bytes, file.footer.offset, chunk)) + for chunk in group.columns] for group in metadata.row_groups] + finally + close(file) + end +end + +mutable struct OITrackedSource <: Parquet.AbstractSource + bytes::Vector{UInt8} + closed::Bool +end + +function Parquet.sourcelength(source::OITrackedSource) + source.closed && throw(ArgumentError("tracked source is closed")) + return Int64(length(source.bytes)) +end + +function Parquet.readrange(source::OITrackedSource, offset::Integer, + count::Integer) + source.closed && throw(ArgumentError("tracked source is closed")) + offset >= 0 || throw(BoundsError(source.bytes, offset)) + count >= 0 || throw(ArgumentError("byte count must be nonnegative")) + stop = Base.checked_add(Int64(offset), Int64(count)) + stop <= length(source.bytes) || throw(BoundsError(source.bytes, + (offset, count))) + first = Int(offset) + 1 + return @view source.bytes[first:(first + Int(count) - 1)] +end + +function Parquet.close!(source::OITrackedSource) + source.closed = true + return +end + +function oireject(bytes::Vector{UInt8}, type::Type{<:Exception}=Parquet.FormatError) + @test_throws type Parquet.Table(bytes) + source = OITrackedSource(copy(bytes), false) + @test_throws type Parquet.Table(source) + @test source.closed + return +end + +function oiframe(payload::Vector{UInt8}; type=OIMD.PageType.INDEX_PAGE, + data=nothing, index=OIMD.IndexPageHeader(), dictionary=nothing, + datav2=nothing, checksum::Bool=true, compressed=length(payload), + uncompressed=compressed) + crc = checksum ? reinterpret(Int32, Parquet.pagechecksum(payload)) : + nothing + header = OIMD.PageHeader(type_=type, + uncompressed_page_size=Int32(uncompressed), + compressed_page_size=Int32(compressed), crc=crc, + data_page_header=data, index_page_header=index, + dictionary_page_header=dictionary, + data_page_header_v2=datav2) + return vcat(OITH.encode(header), payload) +end + +function oidatav1(value::Int32) + payload = Parquet.encode_plain(Int32[value]) + header = OIMD.DataPageHeader(num_values=Int32(1), + encoding=OIMD.Encoding.PLAIN, + definition_level_encoding=OIMD.Encoding.RLE, + repetition_level_encoding=OIMD.Encoding.RLE) + return oiframe(payload; type=OIMD.PageType.DATA_PAGE, data=header, + index=nothing) +end + +function oilegacyfile(frames::Vector{Vector{UInt8}}; + indexposition=nothing, dictionaryposition=nothing) + offsets = Int64[] + parsed = [] + offset = Int64(4) + values = Int64(0) + rows = Int64(0) + locations = OIMD.PageLocation[] + uncompressed = Int64(0) + for raw in frames + source = Parquet.source(raw) + frame = Parquet.readpage(source, Int64(0), Int64(length(raw)), + Parquet.Limits()) + push!(offsets, offset) + push!(parsed, frame.header) + uncompressed = Base.checked_add(uncompressed, + Int64(frame.headerlength) + + Int64(frame.header.uncompressed_page_size)) + if Parquet.pagekind(frame) in (:data_v1, :data_v2) + push!(locations, OIMD.PageLocation(offset=offset, + compressed_page_size=Int32(length(raw)), + first_row_index=rows)) + pagevalues = Parquet._pageentrycount(frame) + values = Base.checked_add(values, pagevalues) + pagerows = frame.header.type_ == OIMD.PageType.DATA_PAGE ? + pagevalues : Int64(frame.header.data_page_header_v2.num_rows) + rows = Base.checked_add(rows, pagerows) + end + offset = Base.checked_add(offset, Int64(length(raw))) + end + datapos = findfirst(header -> header.type_ in + (OIMD.PageType.DATA_PAGE, OIMD.PageType.DATA_PAGE_V2), parsed) + datapos === nothing && throw(ArgumentError("legacy fixture needs data")) + dataoffset = offsets[datapos] + indexoffset = indexposition === nothing ? nothing : + offsets[indexposition] + dictionaryoffset = dictionaryposition === nothing ? nothing : + offsets[dictionaryposition] + column = OIMD.ColumnMetaData(type_=OIMD.Type.INT32, + encodings=OIMD.Encoding.T[OIMD.Encoding.PLAIN, OIMD.Encoding.RLE], + path_in_schema=["value"], codec=OIMD.CompressionCodec.UNCOMPRESSED, + num_values=values, total_uncompressed_size=uncompressed, + total_compressed_size=offset - 4, data_page_offset=dataoffset, + index_page_offset=indexoffset, + dictionary_page_offset=dictionaryoffset) + index = OIMD.OffsetIndex(page_locations=locations) + body = vcat(Parquet.PARQUET_MAGIC, frames...) + indexraw = OITH.encode(index) + indexoffset = Int64(length(body)) + append!(body, indexraw) + chunk = OIMD.ColumnChunk(meta_data=column, + offset_index_offset=indexoffset, + offset_index_length=Int32(length(indexraw))) + group = OIMD.RowGroup(columns=OIMD.ColumnChunk[chunk], + total_byte_size=uncompressed, num_rows=rows, + total_compressed_size=offset - 4, file_offset=Int64(4), + ordinal=Int16(0)) + schema = OIMD.SchemaElement[ + OIMD.SchemaElement(name="schema", num_children=Int32(1)), + OIMD.SchemaElement(name="value", type_=OIMD.Type.INT32, + repetition_type=OIMD.FieldRepetitionType.REQUIRED), + ] + metadata = OIMD.FileMetaData(version=Int32(1), schema=schema, + num_rows=rows, row_groups=OIMD.RowGroup[group], + created_by="Parquet.jl N4-C test") + footer = OITH.encode(metadata) + append!(body, footer) + Parquet._writelittle!(body, UInt32(length(footer))) + append!(body, Parquet.PARQUET_MAGIC) + return body +end + +function oirewritepageheader(transform, bytes::Vector{UInt8}, + offset::Int64) + file = Parquet.File(bytes) + try + frame = Parquet.readpage(file.source, offset, file.footer.offset, + Parquet.Limits()) + header = transform(frame.header) + encoded = OITH.encode(header) + length(encoded) == frame.headerlength || throw(ArgumentError( + "replacement PageHeader changes its serialized length")) + output = copy(bytes) + first = Int(offset) + 1 + copyto!(output, first, encoded, 1, length(encoded)) + return output + finally + close(file) + end +end + +function oierror(f) + try + f() + catch err + return err + end + return nothing +end + +function oiprivateindexfailure(bytes::Vector{UInt8}) + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + Parquet._reservearray!(budget, UInt8, 0) + entry = Parquet._budgetused(budget) + file = Parquet.File(bytes) + try + metadata = oimetadata(file) + schema = Parquet.Schema(metadata) + chunk = metadata.row_groups[1].columns[1] + error = oierror() do + Parquet._readoffsetindex(file, chunk, schema.leaves[1], + metadata.row_groups[1].num_rows, limits, budget) + end + @test error isa Parquet.FormatError + @test Parquet._budgetused(budget) == entry + finally + close(file) + end + return +end + +function oioffsetreadsuccess(bytes::Vector{UInt8}, maximum::Int64) + limits = Parquet.Limits(max_materialized_bytes=maximum) + budget = Parquet._LiveByteBudget(limits) + file = Parquet.File(bytes) + try + metadata = oimetadata(file) + schema = Parquet.Schema(metadata) + Parquet._readoffsetindexes(file, metadata, schema; limits=limits, + budget=budget) + return true + catch err + err isa Parquet.LimitError || rethrow() + return false + finally + close(file) + end +end + +function oiwritesuccess(input, maximum::Int64; pageindex::Bool) + try + Parquet._encodefile(input; pagesize=4, checksum=false, + pageindex=pageindex, limits=Parquet.Limits( + max_materialized_bytes=maximum)) + return true + catch err + err isa Parquet.LimitError || rethrow() + return false + end +end + +function oiminimumwrite(input; pageindex::Bool) + low = Int64(-1) + high = Int64(1_000_000) + oiwritesuccess(input, high; pageindex=pageindex) || + throw(ArgumentError("writer minimum search upper bound is too low")) + while high - low > 1 + middle = (low + high) ÷ 2 + if oiwritesuccess(input, middle; pageindex=pageindex) + high = middle + else + low = middle + end + end + return high +end + +const OICODECS = (:uncompressed, :snappy, :gzip, :brotli, :zstd, + :lz4_raw) + +@testset "offset-index writer default and opt-out" begin + input = oigoldeninput() + default = Parquet._encodefile(input; rowgroupsize=2, pagesize=1) + explicit = Parquet._encodefile(input; rowgroupsize=2, pagesize=1, + pageindex=true) + disabled = Parquet._encodefile(input; rowgroupsize=2, pagesize=1, + pageindex=false) + @test default == explicit + indexed = oiinspect(default) + unindexed = oiinspect(disabled; pageindex=false) + @test [group.num_rows for group in indexed.metadata.row_groups] == + Int64[2, 2, 1] + @test indexed.physicalintervals == unindexed.physicalintervals + lastpage = Int(last(indexed.physicalintervals)[2]) + @test default[1:lastpage] == disabled[1:lastpage] + oiroundtrip(default, input) + oiroundtrip(disabled, input) + + empty = (id=Int32[], days=Vector{Union{Missing,Date}}[]) + emptydefault = Parquet._encodefile(empty) + emptydisabled = Parquet._encodefile(empty; pageindex=false) + @test emptydefault == emptydisabled + emptyresult = oiinspect(emptydefault) + @test emptyresult.metadata.num_rows == 0 + @test isempty(emptyresult.metadata.row_groups) + @test emptyresult.footer_offset == 4 + oiroundtrip(emptydefault, empty) +end + +@testset "offset-index V1 V2 codec and nested row-group matrix" begin + input = oigoldeninput() + expectedrows = (Int64[0, 1], Int64[0, 1], Int64[0]) + expecteddayvalues = (Int64[1, 1], Int64[1, 3], Int64[1]) + for pageversion in (:v1, :v2), codec in OICODECS + bytes = Parquet._encodefile(input; rowgroupsize=2, pagesize=1, + pageversion=pageversion, codec=codec) + result = oiinspect(bytes) + @test length(result.metadata.row_groups) == 3 + for groupindex in 1:3 + for leafindex in 1:2 + locations = result.indexgroups[groupindex][leafindex].page_locations + @test [location.first_row_index for location in locations] == + expectedrows[groupindex] + end + dayframes = filter(oidatapage, + result.groups[groupindex][2]) + @test oipagevalues.(dayframes) == + expecteddayvalues[groupindex] + end + oiroundtrip(bytes, input) + end +end + +@testset "offset-index dictionary pages are excluded" begin + text = "same-value-" * repeat("x", 64) + input = (; value=fill(text, 32)) + for pageversion in (:v1, :v2), codec in OICODECS + bytes = Parquet._encodefile(input; rowgroupsize=16, pagesize=128, + pageversion=pageversion, codec=codec, dictionary=true, + checksum=false) + result = oiinspect(bytes) + @test length(result.metadata.row_groups) == 2 + for groupindex in 1:2 + frames = result.groups[groupindex][1] + @test first(frames).header.type_ == + OIMD.PageType.DICTIONARY_PAGE + locations = only(result.indexgroups[groupindex]).page_locations + @test length(locations) == count(oidatapage, frames) + @test all(location -> location.offset != first(frames).offset, + locations) + @test first(locations).offset == + something(result.metadata.row_groups[groupindex].columns[1].meta_data).data_page_offset + end + oiroundtrip(bytes, input) + end + + mixed = (; value=vcat(fill("same", 128), + ["unique-$(lpad(index, 4, '0'))" for index in 1:128])) + bytes = Parquet._encodefile(mixed; rowgroupsize=128, + pagesize=64, dictionary=true) + result = oiinspect(bytes) + @test result.metadata.row_groups[1].columns[1].meta_data.dictionary_page_offset !== + nothing + @test result.metadata.row_groups[2].columns[1].meta_data.dictionary_page_offset === + nothing + @test all(index -> index !== nothing, Iterators.flatten( + result.indexgroups)) + oiroundtrip(bytes, mixed) +end + +@testset "offset-index footer pairs and global intervals" begin + input = (; left=Int32[1, 2], right=Int32[3, 4]) + bytes = Parquet._encodefile(input; pagesize=4) + metadata = oimetadata(bytes) + firstchunk = metadata.row_groups[1].columns[1] + secondchunk = metadata.row_groups[1].columns[2] + oireject(oichunkrewrite(bytes, 1, 1; offset_index_length=nothing)) + oireject(oichunkrewrite(bytes, 1, 1; offset_index_offset=nothing)) + oireject(oichunkrewrite(bytes, 1, 1; offset_index_length=Int32(0))) + oireject(oichunkrewrite(bytes, 1, 1; offset_index_length=Int32(-1))) + oireject(oichunkrewrite(bytes, 1, 1; offset_index_offset=Int64(-1))) + oireject(oichunkrewrite(bytes, 1, 1; + offset_index_offset=typemax(Int64), offset_index_length=Int32(1))) + file = Parquet.File(bytes) + footer = try + file.footer.offset + finally + close(file) + end + oireject(oichunkrewrite(bytes, 1, 1; offset_index_offset=footer, + offset_index_length=Int32(1))) + firstcolumn = something(firstchunk.meta_data) + firststart = oichunkstart(firstcolumn) + oireject(oichunkrewrite(bytes, 1, 1; + offset_index_offset=firststart, + offset_index_length=firstchunk.offset_index_length)) + overlappingoi = oichunkrewrite(bytes, 1, 2; + offset_index_offset=firstchunk.offset_index_offset, + offset_index_length=firstchunk.offset_index_length) + oireject(overlappingoi) + overlapfile = Parquet.File(overlappingoi) + overlaplimits = Parquet.Limits() + overlapbudget = Parquet._LiveByteBudget(overlaplimits) + Parquet._reservearray!(overlapbudget, UInt8, 0) + overlapentry = Parquet._budgetused(overlapbudget) + try + overlapmetadata = oimetadata(overlapfile) + overlapschema = Parquet.Schema(overlapmetadata) + overlaperror = oierror() do + Parquet._preflightoffsetindexranges(overlapfile, + overlapmetadata, overlapschema, overlaplimits, + overlapbudget) + end + @test overlaperror isa Parquet.FormatError + @test overlaperror.message == + "physical column, page-index, or bloom-filter storage ranges overlap" + @test Parquet._budgetused(overlapbudget) == overlapentry + finally + close(overlapfile) + end + + secondcolumn = something(secondchunk.meta_data) + overlappedmetadata = oimetadatawithchunk(metadata, 1, 2, + oireplace(secondchunk; meta_data=oireplace(secondcolumn; + data_page_offset=firstcolumn.data_page_offset, + total_compressed_size=firstcolumn.total_compressed_size))) + oireject(oirewritefooter(bytes, overlappedmetadata)) + oiroundtrip(bytes, input) +end + +@testset "offset-index Compact Thrift and PageLocation corruption" begin + input = (; value=Int32[1, 2, 3, 4, 5]) + bytes = Parquet._encodefile(input; pagesize=4, checksum=false) + indexes = oiindexobjects(bytes) + index = indexes[1][1] + locations = index.page_locations + @test length(locations) == 5 + + trailing = [Any[vcat(OITH.encode(index), UInt8[0x00])]] + oireject(oirebuildsections(bytes; indexes=trailing)) + truncated = [Any[OITH.encode(index)[1:(end - 1)]]] + oireject(oirebuildsections(bytes; indexes=truncated)) + oireject(oirebuildsections(bytes; indexes=[Any[UInt8[0xff]]])) + oireject(oirebuildsections(bytes; indexes=[Any[ + oireplace(index; page_locations=OIMD.PageLocation[])] ])) + oireject(oirebuildsections(bytes; indexes=[Any[ + oireplace(index; page_locations=locations[1:(end - 1)])] ])) + oireject(oirebuildsections(bytes; indexes=[Any[ + oireplace(index; page_locations=vcat(locations, last(locations)))] ])) + + firstlocation = first(locations) + secondlocation = locations[2] + function oilocationfailure(replacement::OIMD.PageLocation, + position::Int=1) + changed = copy(locations) + changed[position] = replacement + corrupted = oireplace(index; page_locations=changed) + oireject(oirebuildsections(bytes; indexes=[Any[corrupted]])) + return + end + oilocationfailure(oireplace(firstlocation; + compressed_page_size=Int32(0))) + oilocationfailure(oireplace(firstlocation; + compressed_page_size=Int32(-1))) + oilocationfailure(oireplace(firstlocation; + compressed_page_size=firstlocation.compressed_page_size + Int32(1))) + oilocationfailure(oireplace(firstlocation; + compressed_page_size=firstlocation.compressed_page_size - Int32(1))) + oilocationfailure(oireplace(firstlocation; offset=firstlocation.offset + 1)) + oilocationfailure(oireplace(firstlocation; offset=Int64(3))) + oilocationfailure(oireplace(firstlocation; first_row_index=Int64(-1))) + oilocationfailure(oireplace(firstlocation; first_row_index=Int64(1))) + oilocationfailure(oireplace(secondlocation; first_row_index=Int64(0)), 2) + oilocationfailure(oireplace(secondlocation; first_row_index=Int64(5)), 2) + oilocationfailure(oireplace(secondlocation; + offset=firstlocation.offset), 2) + oilocationfailure(oireplace(secondlocation; + offset=firstlocation.offset + + Int64(firstlocation.compressed_page_size) - 1), 2) + + nonbyte = oireplace(index; + unencoded_byte_array_data_bytes=fill(Int64(0), length(locations))) + oireject(oirebuildsections(bytes; indexes=[Any[nonbyte]])) + + strings = (; value=["alpha", "beta", "gamma", "delta"]) + stringbytes = Parquet._encodefile(strings; pagesize=8) + stringindexes = oiindexobjects(stringbytes) + stringindex = stringindexes[1][1] + count = length(stringindex.page_locations) + sized = oireplace(stringindex; + unencoded_byte_array_data_bytes=fill(Int64(5), count)) + valid = oirebuildsections(stringbytes; indexes=[Any[sized]]) + oiroundtrip(valid, strings) + wrongcount = oireplace(stringindex; + unencoded_byte_array_data_bytes=fill(Int64(0), count + 1)) + oireject(oirebuildsections(stringbytes; indexes=[Any[wrongcount]])) + negative = oireplace(stringindex; + unencoded_byte_array_data_bytes=vcat(Int64[-1], + fill(Int64(0), count - 1))) + oireject(oirebuildsections(stringbytes; indexes=[Any[negative]])) + + dictionaryinput = (; value=fill("dictionary-value", 16)) + dictionarybytes = Parquet._encodefile(dictionaryinput; dictionary=true, + pagesize=16) + dictionarymetadata = oimetadata(dictionarybytes) + dictionaryindex = oiindexobjects(dictionarybytes)[1][1] + dictionarylocation = first(dictionaryindex.page_locations) + dictionaryoffset = something(dictionarymetadata.row_groups[1].columns[1].meta_data).dictionary_page_offset + pointsatdictionary = copy(dictionaryindex.page_locations) + pointsatdictionary[1] = oireplace(dictionarylocation; + offset=Int64(dictionaryoffset)) + oireject(oirebuildsections(dictionarybytes; indexes=[Any[ + oireplace(dictionaryindex; page_locations=pointsatdictionary)] ])) +end + +@testset "column-index pairing and opaque interval validation" begin + input = (; left=Int32[1, 2], right=Int32[3, 4]) + bytes = Parquet._encodefile(input; pagesize=4) + columnraws = [Any[fill(UInt8(0xaa), 32), fill(UInt8(0xbb), 16)]] + withcolumns = oirebuildsections(bytes; columns=columnraws) + oiroundtrip(withcolumns, input) + metadata = oimetadata(withcolumns) + firstchunk = metadata.row_groups[1].columns[1] + secondchunk = metadata.row_groups[1].columns[2] + @test firstchunk.column_index_offset !== nothing + @test firstchunk.column_index_length == 32 + @test secondchunk.column_index_length == 16 + + oireject(oichunkrewrite(withcolumns, 1, 1; + column_index_length=nothing)) + oireject(oichunkrewrite(withcolumns, 1, 1; + column_index_offset=nothing)) + oireject(oichunkrewrite(withcolumns, 1, 1; + column_index_length=Int32(0))) + oireject(oichunkrewrite(withcolumns, 1, 1; + column_index_length=Int32(-1))) + oireject(oichunkrewrite(withcolumns, 1, 1; + column_index_offset=typemax(Int64), column_index_length=Int32(1))) + oireject(oichunkrewrite(withcolumns, 1, 1; + offset_index_offset=nothing, offset_index_length=nothing)) + + firstcolumn = something(firstchunk.meta_data) + oireject(oichunkrewrite(withcolumns, 1, 1; + column_index_offset=oichunkstart(firstcolumn), + column_index_length=Int32(1))) + oireject(oichunkrewrite(withcolumns, 1, 1; + column_index_offset=firstchunk.offset_index_offset, + column_index_length=Int32(1))) + oireject(oichunkrewrite(withcolumns, 1, 2; + column_index_offset=firstchunk.column_index_offset, + column_index_length=secondchunk.column_index_length)) +end + +@testset "legacy INDEX_PAGE and unknown framed pages" begin + data1 = oidatav1(Int32(1)) + data2 = oidatav1(Int32(2)) + index = oiframe(UInt8[0x10, 0x20]) + unknown = oiframe(UInt8[0x30, 0x40]; type=OIMD.PageType.T(9), + index=nothing) + + leading = oilegacyfile([index, data1]; indexposition=1) + oiroundtrip(leading, (; value=Int32[1])) + leadingmetadata = oimetadata(leading) + leadingchunk = leadingmetadata.row_groups[1].columns[1] + @test leadingchunk.meta_data.index_page_offset == 4 + @test first(oiindexobjects(leading)[1][1].page_locations).offset == + 4 + length(index) + + interleaved = oilegacyfile([data1, index, data2]; indexposition=2) + oiroundtrip(interleaved, (; value=Int32[1, 2])) + interleavedindex = oiindexobjects(interleaved)[1][1] + @test [location.offset for location in interleavedindex.page_locations] == + Int64[4, 4 + length(data1) + length(index)] + @test all(location -> location.offset != 4 + length(data1), + interleavedindex.page_locations) + + unknownfile = oilegacyfile([data1, unknown, data2]) + oiroundtrip(unknownfile, (; value=Int32[1, 2])) + unknownindex = oiindexobjects(unknownfile)[1][1] + @test [location.offset for location in unknownindex.page_locations] == + Int64[4, 4 + length(data1) + length(unknown)] + wrongunknown = copy(unknownindex.page_locations) + wrongunknown[2] = oireplace(wrongunknown[2]; + offset=Int64(4 + length(data1))) + oireject(oirebuildsections(unknownfile; indexes=[Any[ + oireplace(unknownindex; page_locations=wrongunknown)] ])) + + dictionary = oiframe(Parquet.encode_plain(Int32[9]); + type=OIMD.PageType.DICTIONARY_PAGE, index=nothing, + dictionary=OIMD.DictionaryPageHeader(num_values=Int32(1), + encoding=OIMD.Encoding.PLAIN)) + dictionaryfirst = oilegacyfile([dictionary, index, data1]; + indexposition=2, dictionaryposition=1) + oiroundtrip(dictionaryfirst, (; value=Int32[1])) + indexbeforedictionary = oilegacyfile([index, dictionary, data1]; + indexposition=1, dictionaryposition=2) + oireject(indexbeforedictionary) + + validtail = oilegacyfile([data1, index]; indexposition=2) + corruptcrc = copy(validtail) + corruptcrc[4 + length(data1) + length(index)] ⊻= 0x01 + oireject(corruptcrc) + indexoffset = Int64(4 + length(data1)) + truncated = oirewritepageheader(validtail, indexoffset) do header + return oireplace(header; + compressed_page_size=header.compressed_page_size + Int32(1)) + end + oireject(truncated) + oireject(oichunkrewrite(leading, 1, 1; + meta_data=oireplace(something(leadingchunk.meta_data); + index_page_offset=Int64(0)))) + oireject(oichunkrewrite(leading, 1, 1; + meta_data=oireplace(something(leadingchunk.meta_data); + index_page_offset=Int64(-1)))) + oireject(oichunkrewrite(leading, 1, 1; + meta_data=oireplace(something(leadingchunk.meta_data); + index_page_offset=something(leadingchunk.meta_data).data_page_offset))) +end + +@testset "physical frame limits include skipped and dictionary pages" begin + index = oiframe(UInt8[0x10]) + unknown = oiframe(UInt8[0x20]; type=OIMD.PageType.T(9), + index=nothing) + dictionary = oiframe(Parquet.encode_plain(Int32[9]); + type=OIMD.PageType.DICTIONARY_PAGE, index=nothing, + dictionary=OIMD.DictionaryPageHeader(num_values=Int32(1), + encoding=OIMD.Encoding.PLAIN)) + function framecountdatav2(value::Int32) + return oiframe(Parquet.encode_plain(Int32[value]); + type=OIMD.PageType.DATA_PAGE_V2, index=nothing, + datav2=OIMD.DataPageHeaderV2(num_values=Int32(1), + num_nulls=Int32(0), num_rows=Int32(1), + encoding=OIMD.Encoding.PLAIN, + definition_levels_byte_length=Int32(0), + repetition_levels_byte_length=Int32(0), + is_compressed=false)) + end + function offsetframeerror(bytes::Vector{UInt8}, limits::Parquet.Limits) + file = Parquet.File(bytes) + try + metadata = oimetadata(file) + schema = Parquet.Schema(metadata) + chunk = metadata.row_groups[1].columns[1] + indexobject, _ = oirawindex(bytes, file.footer.offset, chunk) + budget = Parquet._LiveByteBudget(limits) + return oierror() do + Parquet._validateoffsetindexframes(file, chunk, + schema.leaves[1], metadata.row_groups[1].num_rows, + indexobject, limits, budget) + end + finally + close(file) + end + end + for data in ((oidatav1(Int32(1)), oidatav1(Int32(2))), + (framecountdatav2(Int32(1)), framecountdatav2(Int32(2)))) + frames = [index, unknown, data[1], data[2], unknown] + bytes = oilegacyfile(frames; indexposition=1) + table = Parquet.Table(bytes; limits=Parquet.Limits( + max_container_elements=5)) + @test table.columns.value == Int32[1, 2] + close(table) + failure = oierror() do + Parquet.Table(bytes; limits=Parquet.Limits( + max_container_elements=4)) + end + @test failure isa Parquet.LimitError + @test failure.resource == :container_elements + @test failure.requested == 5 + @test failure.maximum == 4 + + dictionaryframes = [dictionary, index, unknown, data[1], data[2], + unknown] + dictionarybytes = oilegacyfile(dictionaryframes; indexposition=2, + dictionaryposition=1) + table = Parquet.Table(dictionarybytes; limits=Parquet.Limits( + max_container_elements=6)) + @test table.columns.value == Int32[1, 2] + close(table) + failure = oierror() do + Parquet.Table(dictionarybytes; limits=Parquet.Limits( + max_container_elements=5)) + end + @test failure isa Parquet.LimitError + @test failure.requested == 6 + @test failure.maximum == 5 + end + + frames = [dictionary, index, unknown, oidatav1(Int32(1)), + oidatav1(Int32(2)), unknown] + bytes = oilegacyfile(frames; indexposition=2, dictionaryposition=1) + file = Parquet.File(bytes) + limits = Parquet.Limits(max_container_elements=5) + budget = Parquet._LiveByteBudget(limits) + Parquet._reservearray!(budget, UInt8, 0) + entry = Parquet._budgetused(budget) + try + metadata = oimetadata(file) + schema = Parquet.Schema(metadata) + chunk = metadata.row_groups[1].columns[1] + indexobject = oiindexobjects(bytes)[1][1] + failure = oierror() do + Parquet._validateoffsetindexframes(file, chunk, + schema.leaves[1], metadata.row_groups[1].num_rows, + indexobject, limits, budget) + end + @test failure isa Parquet.LimitError + @test failure.requested == 6 + @test Parquet._budgetused(budget) == entry + finally + close(file) + end + + precedenceframes = [index, unknown, unknown, unknown, + oidatav1(Int32(1))] + precedencebytes = oilegacyfile(precedenceframes; indexposition=1) + precedenceindex = oiindexobjects(precedencebytes)[1][1] + badlocations = copy(precedenceindex.page_locations) + badlocations[1] = oireplace(badlocations[1]; + offset=badlocations[1].offset + 1) + badoffset = oirebuildsections(precedencebytes; indexes=[Any[ + oireplace(precedenceindex; page_locations=badlocations)]]) + precedenceerror = oierror() do + Parquet.Table(badoffset; limits=Parquet.Limits( + max_container_elements=4)) + end + @test precedenceerror isa Parquet.FormatError + + @test Parquet._rangesoverlap((Int64(0), typemax(Int64)), + (typemax(Int64) - 1, Int64(1))) + rangeerror = oierror() do + Parquet._rangesoverlap((typemax(Int64), Int64(1)), + (Int64(0), Int64(1))) + end + @test rangeerror isa Parquet.FormatError + @test rangeerror.message == "page-index interval range overflows Int64" + @test_throws Parquet.FormatError Parquet._rangesoverlap( + (Int64(-1), Int64(1)), (Int64(0), Int64(1))) + @test_throws Parquet.FormatError Parquet._rangesoverlap( + (Int64(0), Int64(-1)), (Int64(0), Int64(1))) + + intervallimits = Parquet.Limits(max_container_elements=3) + @test Parquet._pageindexintervalcount(Int64(1), Int64(2), + intervallimits) == 3 + intervalerror = oierror() do + Parquet._pageindexintervalcount(Int64(2), Int64(2), + intervallimits) + end + @test intervalerror isa Parquet.LimitError + @test intervalerror.resource == :container_elements + @test intervalerror.requested == 4 + @test intervalerror.maximum == 3 + intervaloverflow = oierror() do + Parquet._pageindexintervalcount(typemax(Int64), Int64(1), + Parquet.Limits(max_container_elements=typemax(Int64))) + end + @test intervaloverflow isa Parquet.LimitError + @test intervaloverflow.requested == typemax(Int64) + + negative = oiframe(UInt8[]; + type=OIMD.PageType.DICTIONARY_PAGE, index=nothing, + dictionary=OIMD.DictionaryPageHeader(num_values=Int32(-1), + encoding=OIMD.Encoding.PLAIN)) + negativeerror = offsetframeerror(oilegacyfile( + [negative, oidatav1(Int32(1))]; dictionaryposition=1), + Parquet.Limits(max_container_elements=0)) + @test negativeerror isa Parquet.FormatError + @test negativeerror.message == "negative page value count" + + invalidencoding = oiframe(Parquet.encode_plain(Int32[9, 10]); + type=OIMD.PageType.DICTIONARY_PAGE, index=nothing, + dictionary=OIMD.DictionaryPageHeader(num_values=Int32(2), + encoding=OIMD.Encoding.RLE)) + encodingerror = offsetframeerror(oilegacyfile( + [invalidencoding, oidatav1(Int32(1))]; dictionaryposition=1), + Parquet.Limits(max_container_elements=1)) + @test encodingerror isa Parquet.FormatError + @test occursin("is not PLAIN", encodingerror.message) + + oversizeddictionary = oiframe(Parquet.encode_plain(Int32[9, 10]); + type=OIMD.PageType.DICTIONARY_PAGE, index=nothing, + dictionary=OIMD.DictionaryPageHeader(num_values=Int32(2), + encoding=OIMD.Encoding.PLAIN)) + entryerror = offsetframeerror(oilegacyfile( + [oversizeddictionary, oidatav1(Int32(1))]; dictionaryposition=1), + Parquet.Limits(max_container_elements=1)) + @test entryerror isa Parquet.LimitError + @test entryerror.resource == :container_elements + @test entryerror.requested == 2 + @test entryerror.maximum == 1 + + emptydictionary = oiframe(UInt8[]; + type=OIMD.PageType.DICTIONARY_PAGE, index=nothing, + dictionary=OIMD.DictionaryPageHeader(num_values=Int32(0), + encoding=OIMD.Encoding.PLAIN)) + frameerror = offsetframeerror(oilegacyfile( + [emptydictionary, oidatav1(Int32(1))]; dictionaryposition=1), + Parquet.Limits(max_container_elements=0)) + @test frameerror isa Parquet.LimitError + @test frameerror.resource == :container_elements + @test frameerror.requested == 1 + @test frameerror.maximum == 0 +end + +@testset "offset-index V1 V2 row and value semantics" begin + E = Union{Missing,Date} + nested = (; days=Union{Missing,Vector{E}}[ + E[Date(2000, 1, 1), Date(2001, 1, 1), Date(2002, 1, 1)], + E[Date(2003, 1, 1)], + E[Date(2004, 1, 1)], + ]) + v1 = Parquet._encodefile(nested; pageversion=:v1, pagesize=40, + rowgroupsize=nothing, checksum=false) + v1result = oiinspect(v1) + @test [location.first_row_index for location in + v1result.indexgroups[1][1].page_locations] == Int64[0, 1] + @test [oipagerows(frame, + something(v1result.metadata.row_groups[1].columns[1].meta_data), + only(v1result.schema.leaves)) for frame in + filter(oidatapage, v1result.groups[1][1])] == Int64[1, 2] + oiroundtrip(v1, nested) + + v2 = Parquet._encodefile(nested; pageversion=:v2, pagesize=40, + rowgroupsize=nothing, checksum=false) + result = oiinspect(v2) + frames = filter(oidatapage, result.groups[1][1]) + @test [frame.header.data_page_header_v2.num_rows for frame in frames] == + Int32[1, 2] + swapped = oirewritepageheader(v2, frames[1].offset) do header + page = oireplace(header.data_page_header_v2; num_rows=Int32(2)) + return oireplace(header; data_page_header_v2=page) + end + swapped = oirewritepageheader(swapped, frames[2].offset) do header + page = oireplace(header.data_page_header_v2; num_rows=Int32(1)) + return oireplace(header; data_page_header_v2=page) + end + swappedindex = oiindexobjects(swapped)[1][1] + swappedlocations = copy(swappedindex.page_locations) + swappedlocations[2] = oireplace(swappedlocations[2]; + first_row_index=Int64(2)) + oireject(oirebuildsections(swapped; indexes=[Any[ + oireplace(swappedindex; page_locations=swappedlocations)] ])) + + changedvalues = oirewritepageheader(v2, frames[1].offset) do header + page = oireplace(header.data_page_header_v2; + num_values=header.data_page_header_v2.num_values + Int32(1)) + return oireplace(header; data_page_header_v2=page) + end + oireject(changedvalues) + + flat = Parquet._encodefile((value=Int32[1, 2, 3],); + pageversion=:v2, pagesize=nothing, checksum=false) + flatresult = oiinspect(flat) + flatoffset = only(only(flatresult.indexgroups)).page_locations[1].offset + wrongflat = oirewritepageheader(flat, flatoffset) do header + page = oireplace(header.data_page_header_v2; num_rows=Int32(2)) + return oireplace(header; data_page_header_v2=page) + end + oireject(wrongflat) +end + +@testset "offset-index cumulative and shared materialized limits" begin + probe = (; value=Int32.(1:64)) + indexed = Parquet._encodefile(probe; pagesize=4, checksum=false) + metadata = oimetadata(indexed) + lengths = Int64[chunk.offset_index_length for group in + metadata.row_groups for chunk in group.columns] + total = sum(lengths; init=Int64(0)) + @test total > 0 + @test Parquet._encodefile(probe; pagesize=4, checksum=false, + limits=Parquet.Limits(max_page_index_bytes=total)) == indexed + writerlimit = oierror() do + Parquet._encodefile(probe; pagesize=4, checksum=false, + limits=Parquet.Limits(max_page_index_bytes=total - 1)) + end + @test writerlimit isa Parquet.LimitError + @test writerlimit.resource == :page_index_bytes + table = Parquet.Table(indexed; + limits=Parquet.Limits(max_page_index_bytes=total)) + close(table) + readerlimit = oierror() do + Parquet.Table(indexed; + limits=Parquet.Limits(max_page_index_bytes=total - 1)) + end + @test readerlimit isa Parquet.LimitError + @test readerlimit.resource == :page_index_bytes + + twocolumns = Parquet._encodefile((left=probe.value, + right=reverse(probe.value)); pagesize=4, checksum=false) + twometadata = oimetadata(twocolumns) + twolengths = Int64[chunk.offset_index_length for group in + twometadata.row_groups for chunk in group.columns] + @test length(twolengths) == 2 + @test sum(twolengths) > maximum(twolengths) + cumulative = oierror() do + Parquet.Table(twocolumns; limits=Parquet.Limits( + max_page_index_bytes=maximum(twolengths))) + end + @test cumulative isa Parquet.LimitError + @test cumulative.resource == :page_index_bytes + table = Parquet.Table(twocolumns; limits=Parquet.Limits( + max_page_index_bytes=sum(twolengths))) + close(table) + + preflightlimits = Parquet.Limits() + preflightbudget = Parquet._LiveByteBudget(preflightlimits) + Parquet._reservearray!(preflightbudget, UInt8, 0) + preflightentry = Parquet._budgetused(preflightbudget) + preflightfile = Parquet.File(indexed) + try + preflightschema = Parquet.Schema(metadata) + _, cumulativebytes, rangecharge = + Parquet._preflightoffsetindexranges(preflightfile, metadata, + preflightschema, preflightlimits, preflightbudget) + rangetype = Union{Nothing,Tuple{Int64,Int64}} + expectedrange = Parquet._materializedarraybytes( + Vector{rangetype}, length(metadata.row_groups)) + for group in metadata.row_groups + expectedrange = Parquet._materializedsum(expectedrange, + Parquet._materializedarraybytes(rangetype, + length(group.columns))) + end + @test cumulativebytes == total + @test rangecharge == expectedrange + @test Parquet._budgetused(preflightbudget) - preflightentry == + expectedrange + Parquet._release!(preflightbudget, rangecharge) + @test Parquet._budgetused(preflightbudget) == preflightentry + finally + close(preflightfile) + end + + rangetype = Union{Nothing,Tuple{Int64,Int64}} + entrycharge = Parquet._materializedarraybytes(UInt8, 0) + outercharge = Parquet._materializedarraybytes( + Vector{rangetype}, length(metadata.row_groups)) + intervalcount = Int64(2) + intervalcharge = Parquet._materializedarraybytes( + Parquet._PageIndexInterval, intervalcount) + failurelimits = Parquet.Limits(max_materialized_bytes= + entrycharge + outercharge + intervalcharge) + failurebudget = Parquet._LiveByteBudget(failurelimits) + Parquet._reservearray!(failurebudget, UInt8, 0) + failureentry = Parquet._budgetused(failurebudget) + failurefile = Parquet.File(indexed) + try + failureschema = Parquet.Schema(metadata) + budgeterror = oierror() do + Parquet._preflightoffsetindexranges(failurefile, metadata, + failureschema, failurelimits, failurebudget) + end + @test budgeterror isa Parquet.LimitError + @test budgeterror.resource == :materialized_bytes + @test Parquet._budgetused(failurebudget) == failureentry + finally + close(failurefile) + end + + readminimum = Int64(0) + while !oioffsetreadsuccess(indexed, readminimum) + readminimum = iszero(readminimum) ? Int64(1) : 2 * readminimum + end + readlow = Int64(-1) + readhigh = readminimum + while readhigh - readlow > 1 + middle = (readlow + readhigh) ÷ 2 + if oioffsetreadsuccess(indexed, middle) + readhigh = middle + else + readlow = middle + end + end + @test oioffsetreadsuccess(indexed, readhigh) + @test !oioffsetreadsuccess(indexed, readhigh - 1) + + falseminimum = oiminimumwrite(probe; pageindex=false) + trueminimum = oiminimumwrite(probe; pageindex=true) + @test trueminimum > falseminimum + @test oiwritesuccess(probe, falseminimum; pageindex=false) + @test !oiwritesuccess(probe, falseminimum - 1; pageindex=false) + @test !oiwritesuccess(probe, falseminimum; pageindex=true) + @test oiwritesuccess(probe, trueminimum; pageindex=true) + @test !oiwritesuccess(probe, trueminimum - 1; pageindex=true) + + index = oiindexobjects(indexed)[1][1] + exact = Parquet._writeoffsetindexencodedsize(index) + insufficient = Parquet.Limits(max_materialized_bytes=exact + 64) + insufficientbudget = Parquet._LiveByteBudget(insufficient) + Parquet._reservearray!(insufficientbudget, UInt8, 0) + entry = Parquet._budgetused(insufficientbudget) + peakfailure = oierror() do + Parquet._writeencodeoffsetindex(index, Int64(0), insufficient, + insufficientbudget) + end + @test peakfailure isa Parquet.LimitError + @test peakfailure.resource == :materialized_bytes + @test Parquet._budgetused(insufficientbudget) == entry + charge = Parquet._materializedsum( + Parquet._materializedarraybytes(UInt8, exact), + Parquet._MATERIALIZED_OBJECT_BYTES) + sufficient = Parquet.Limits(max_materialized_bytes=entry + charge) + sufficientbudget = Parquet._LiveByteBudget(sufficient) + Parquet._reservearray!(sufficientbudget, UInt8, 0) + encoded, live, cumulativebytes = Parquet._writeencodeoffsetindex(index, + Int64(0), sufficient, sufficientbudget) + @test length(encoded) == exact == cumulativebytes + @test live == charge + Parquet._release!(sufficientbudget, live) + @test Parquet._budgetused(sufficientbudget) == 64 +end + +@testset "offset-index private rollback and public failure atomicity" begin + input = (; value=Int32[1, 2, 3, 4]) + bytes = Parquet._encodefile(input; pagesize=4, checksum=false) + index = oiindexobjects(bytes)[1][1] + malformed = oirebuildsections(bytes; indexes=[Any[UInt8[0xff]]]) + oiprivateindexfailure(malformed) + trailing = oirebuildsections(bytes; indexes=[Any[ + vcat(OITH.encode(index), UInt8[0x00])]]) + oiprivateindexfailure(trailing) + locations = copy(index.page_locations) + locations[1] = oireplace(locations[1]; offset=locations[1].offset + 1) + badframe = oirebuildsections(bytes; indexes=[Any[ + oireplace(index; page_locations=locations)]]) + oiprivateindexfailure(badframe) + + limits = Parquet.Limits(max_page_index_bytes=1) + io = IOBuffer() + error = oierror() do + Parquet.write(io, input; pagesize=4, limits=limits) + end + @test error isa Parquet.LimitError + @test isempty(take!(io)) + mktempdir() do directory + newpath = joinpath(directory, "new.parquet") + error = oierror() do + Parquet.write(newpath, input; pagesize=4, limits=limits) + end + @test error isa Parquet.LimitError + @test !ispath(newpath) + existing = joinpath(directory, "existing.parquet") + sentinel = UInt8[0x73, 0x61, 0x66, 0x65] + open(existing, "w") do output + write(output, sentinel) + end + error = oierror() do + Parquet.write(existing, input; pagesize=4, limits=limits) + end + @test error isa Parquet.LimitError + @test read(existing) == sentinel + end +end + +@testset "column-index bytes are outside page-index budgets" begin + input = (; left=Int32[1, 2], right=Int32[3, 4]) + bytes = Parquet._encodefile(input; pagesize=4) + metadata = oimetadata(bytes) + total = sum(Int64(chunk.offset_index_length) for group in + metadata.row_groups for chunk in group.columns; init=Int64(0)) + columnraws = [Any[fill(UInt8(0xaa), 16_384), + fill(UInt8(0xbb), 16_384)]] + withcolumns = oirebuildsections(bytes; columns=columnraws) + table = Parquet.Table(withcolumns; limits=Parquet.Limits( + max_page_index_bytes=total)) + try + @test table.columns == input + finally + close(table) + end +end diff --git a/test/write_provenance.jl b/test/write_provenance.jl new file mode 100644 index 0000000..89b4750 --- /dev/null +++ b/test/write_provenance.jl @@ -0,0 +1,692 @@ +using Test + +const WPMD = Parquet.Metadata +const WPTH = Parquet.Thrift +const WRITE_PROVENANCE_CORPUS = get(ENV, "PARQUET_TESTING_DIR", + joinpath(@__DIR__, "parquet-testing")) + +struct WPNullKeyVector <: AbstractVector{String} + values::Vector{String} +end + +function Base.IndexStyle(::Type{WPNullKeyVector}) + return IndexLinear() +end + +function Base.size(values::WPNullKeyVector) + return size(values.values) +end + +function Base.getindex(values::WPNullKeyVector, index::Int) + return index == firstindex(values.values) ? missing : values.values[index] +end + +function wpfixture(name::AbstractString) + return joinpath(WRITE_PROVENANCE_CORPUS, "data", name) +end + +function wpreplace(value; replacements...) + names = fieldnames(typeof(value)) + fields = map(names) do name + return haskey(replacements, name) ? replacements[name] : getfield(value, name) + end + return typeof(value)(fields...) +end + +function wpmetadata(bytes::AbstractVector{UInt8}) + file = Parquet.File(bytes) + try + return WPTH.decode(copy(file.footer.bytes), WPMD.FileMetaData) + finally + close(file) + end +end + +function wprewritefooter(bytes::Vector{UInt8}, metadata::WPMD.FileMetaData) + file = Parquet.File(bytes) + offset = try + file.footer.offset + finally + close(file) + end + output = copy(bytes[1:Int(offset)]) + footer = WPTH.encode(metadata) + append!(output, footer) + Parquet._writelittle!(output, UInt32(length(footer))) + append!(output, Parquet.PARQUET_MAGIC) + return output +end + +function wprawi32(id::Integer, value::Integer) + writer = WPTH.Writer() + WPTH.writei32!(writer, Int32(value)) + return WPTH.RawField(id, WPTH.I32, writer.buffer) +end + +function wpschemaequal(left, right) + length(left) == length(right) || return false + for index in eachindex(left, right) + Parquet._provenanceexact(left[index], right[index]) || return false + end + return true +end + +function wpsemantic(value) + ismissing(value) && return missing + if value isa Parquet.StructValue + output = Pair{String,Any}[] + sizehint!(output, length(value)) + for index in 1:length(value) + push!(output, value.names[index] => wpsemantic(value[index])) + end + return output + end + if value isa Parquet.MapValue + output = Pair{Any,Any}[] + sizehint!(output, length(value)) + for pair in value + push!(output, wpsemantic(pair.first) => wpsemantic(pair.second)) + end + return output + end + if value isa Parquet.ListValue + output = [] + sizehint!(output, length(value)) + for item in value + push!(output, wpsemantic(item)) + end + return output + end + return value +end + +function wpsemanticcolumns(table::Parquet.Table) + names = keys(table.columns) + columns = map(values(table.columns)) do column + return Any[wpsemantic(value) for value in column] + end + return NamedTuple{names}(columns) +end + +function wpexactrewrite(table::Parquet.Table; kwargs...) + expected_schema = table.metadata.schema + expected_values = wpsemanticcolumns(table) + output = Parquet._encodefile(table; kwargs...) + metadata = wpmetadata(output) + @test wpschemaequal(metadata.schema, expected_schema) + rewritten = Parquet.Table(output) + try + @test isequal(wpsemanticcolumns(rewritten), expected_values) + @test rewritten.rows == table.rows + finally + close(rewritten) + end + return output +end + +function wpprovenancesource() + structs = Union{Missing,NamedTuple{(:id,:name),Tuple{Int32,String}}}[ + missing, + (id=Int32(1), name="one"), + (id=Int32(2), name="two"), + ] + lists = Union{Missing,Vector{Union{Missing,Int32}}}[ + missing, + Union{Missing,Int32}[], + Union{Missing,Int32}[Int32(1), missing], + ] + maps = Union{Missing,Dict{String,Union{Missing,Int32}}}[ + missing, + Dict{String,Union{Missing,Int32}}(), + Dict{String,Union{Missing,Int32}}("a" => Int32(1), "b" => missing), + ] + return (; s=structs, l=lists, m=maps) +end + +function wpfreshtable() + return Parquet.Table(Parquet._encodefile(wpprovenancesource())) +end + +function wpreplacecolumn!(table::Parquet.Table, name::Symbol, replacement) + names = keys(table.columns) + columns = ntuple(length(names)) do index + return names[index] == name ? replacement : values(table.columns)[index] + end + updated = NamedTuple{names}(columns) + typeof(updated) === typeof(table.columns) || throw(ArgumentError( + "replacement changed the concrete table column type")) + table.columns = updated + return table +end + +function wprejects(f) + error = try + f() + nothing + catch err + err + end + @test error !== nothing + @test error isa Union{ArgumentError,Parquet.FormatError,Parquet.LimitError} + return error +end + +function wpdeepstructtable(depth::Int, rows::Int) + depth >= 1 || throw(ArgumentError("deep struct depth must be positive")) + rows in (0, 1) || throw(ArgumentError("deep struct rows must be zero or one")) + required = WPMD.FieldRepetitionType.REQUIRED + elements = Vector{WPMD.SchemaElement}(undef, depth + 2) + elements[1] = WPMD.SchemaElement(name="schema", + num_children=Int32(1)) + for level in 1:depth + name = level == 1 ? "deep" : "level_$level" + elements[level + 1] = WPMD.SchemaElement(name=name, + repetition_type=required, num_children=Int32(1)) + end + elements[end] = WPMD.SchemaElement(name="value", + type_=WPMD.Type.INT32, repetition_type=required) + limits = Parquet.Limits(max_metadata_depth=depth + 2, + max_container_elements=max(10_000, depth + 2)) + schema = Parquet.Schema(elements; limits=limits) + column::AbstractVector = iszero(rows) ? Int32[] : Int32[7] + for level in depth:-1:1 + childname = level == depth ? "value" : "level_$(level + 1)" + column = Parquet.StructVector(String[childname], + AbstractVector[column]; rows=rows) + end + seed = Parquet._encodefile((seed=Int32[1],)) + file = Parquet.File(seed) + metadata = WPTH.decode(copy(file.footer.bytes), WPMD.FileMetaData) + metadata = wpreplace(metadata; schema=elements, num_rows=Int64(rows), + row_groups=WPMD.RowGroup[]) + table = Parquet.Table(file, metadata, schema, (; deep=column), rows, false) + return table, limits +end + +function wpdeepstructkeytable(depth::Int) + depth >= 1 || throw(ArgumentError("deep key depth must be positive")) + required = WPMD.FieldRepetitionType.REQUIRED + repeated = WPMD.FieldRepetitionType.REPEATED + elements = Vector{WPMD.SchemaElement}(undef, depth + 4) + elements[1] = WPMD.SchemaElement(name="schema", num_children=Int32(1)) + elements[2] = WPMD.SchemaElement(name="m", + repetition_type=required, num_children=Int32(1), + converted_type=WPMD.ConvertedType.MAP, + logicalType=WPMD.LogicalType(MAP=WPMD.MapType())) + elements[3] = WPMD.SchemaElement(name="key_value", + repetition_type=repeated, num_children=Int32(1)) + for level in 1:depth + name = level == 1 ? "key" : "level_$level" + elements[level + 3] = WPMD.SchemaElement(name=name, + repetition_type=required, num_children=Int32(1)) + end + elements[end] = WPMD.SchemaElement(name="value", + type_=WPMD.Type.INT32, repetition_type=required) + limits = Parquet.Limits(max_metadata_depth=depth + 4, + max_container_elements=max(10_000, depth + 4)) + schema = Parquet.Schema(elements; limits=limits) + key::AbstractVector = Int32[7] + for level in depth:-1:1 + childname = level == depth ? "value" : "level_$(level + 1)" + key = Parquet.StructVector(String[childname], AbstractVector[key]; + rows=1) + end + column = Parquet.MapVector(Int32[0, 1], key, nothing) + seed = Parquet._encodefile((seed=Int32[1],)) + file = Parquet.File(seed) + metadata = WPMD.FileMetaData(version=Int32(1), schema=elements, + num_rows=Int64(1), row_groups=WPMD.RowGroup[]) + table = Parquet.Table(file, metadata, schema, (; m=column), 1, false) + return table, limits +end + +function wpbindingpass(table, semantic, limits, budget, topology) + before = Parquet._budgetused(budget) + bindings = Parquet._provenancebindings(table, semantic, limits, budget, + topology) + count = length(bindings) + Parquet._release!(budget, Parquet._budgetused(budget) - before) + return count +end + +function wpprovenanceallocations(name::Symbol) + source = NamedTuple{(name,)}((Int32[1],)) + table = Parquet.Table(Parquet._encodefile(source)) + try + limits = Parquet.Limits() + budget = Parquet._LiveByteBudget(limits) + _, schema = Parquet._provenancefreshschema(table, limits, budget) + semantic = Parquet._nestedplan(schema; limits=limits, budget=budget) + topology = Parquet._provenancetopology(table, limits, budget) + wpbindingpass(table, semantic, limits, budget, topology) + GC.gc() + bindingbytes = @allocated wpbindingpass(table, semantic, limits, budget, + topology) + bindings = Parquet._provenancebindings(table, semantic, limits, budget, + topology) + before = Parquet._budgetused(budget) + Parquet._provenancevalidatetop(table, semantic, bindings, 1, limits, + topology, budget) + Parquet._budgetused(budget) == before || throw(AssertionError( + "provenance validation retained scratch budget")) + GC.gc() + validationbytes = @allocated Parquet._provenancevalidatetop(table, + semantic, bindings, 1, limits, topology, budget) + Parquet._budgetused(budget) == before || throw(AssertionError( + "provenance validation retained measured scratch budget")) + return (; bindingbytes, validationbytes) + finally + close(table) + end +end + +function wpcustomschemafile() + labels = Union{Missing,Vector{Union{Missing,String}}}[ + missing, + Union{Missing,String}[], + Union{Missing,String}["alpha", missing], + ] + amounts = Union{Missing,Parquet.Decimal}[ + Parquet.Decimal(123, 2), missing, Parquet.Decimal(-4, 2)] + decimal = Parquet.LogicalColumn(amounts, :decimal; precision=9, scale=2) + bytes = Parquet._encodefile((; labels, amount=decimal)) + metadata = wpmetadata(bytes) + schema = copy(metadata.schema) + root = schema[1] + schema[1] = wpreplace(root; name="provenance root", field_id=Int32(101), + unknown_fields=(root.unknown_fields..., wprawi32(90, 900))) + labelsindex = findfirst(element -> element.name == "labels", schema) + amountindex = findfirst(element -> element.name == "amount", schema) + stringindex = findfirst(element -> element.type_ == WPMD.Type.BYTE_ARRAY && + element.logicalType !== nothing && element.logicalType.STRING !== nothing, schema) + for (index, fieldid, rawid) in ((labelsindex, 102, 91), + (stringindex, 103, 92), (amountindex, 104, 93)) + element = schema[index] + schema[index] = wpreplace(element; field_id=Int32(fieldid), + unknown_fields=(element.unknown_fields..., wprawi32(rawid, rawid * 10))) + end + custom = wpreplace(metadata; schema=schema) + return wprewritefooter(bytes, custom) +end + +@testset "schema-bearing writer exact provenance" begin + bytes = wpcustomschemafile() + table = Parquet.Table(bytes) + expected = table.metadata.schema + @test expected[1].name == "provenance root" + @test expected[1].field_id == Int32(101) + @test !isempty(expected[1].unknown_fields) + list = only(filter(element -> element.name == "labels", expected)) + @test list.converted_type == WPMD.ConvertedType.LIST + @test list.logicalType.LIST !== nothing + string = only(filter(element -> element.type_ == WPMD.Type.BYTE_ARRAY && + element.logicalType !== nothing && element.logicalType.STRING !== nothing, + expected)) + @test string.converted_type == WPMD.ConvertedType.UTF8 + decimal = only(filter(element -> element.name == "amount", expected)) + @test decimal.converted_type == WPMD.ConvertedType.DECIMAL + @test decimal.logicalType.DECIMAL.precision == Int32(9) + @test decimal.logicalType.DECIMAL.scale == Int32(2) + @test decimal.precision == Int32(9) + @test decimal.scale == Int32(2) + close(table) + @test @atomic table.closed + for pageversion in (:v1, :v2), codec in (:uncompressed, :snappy) + output = wpexactrewrite(table; pageversion=pageversion, codec=codec) + @test wpschemaequal(wpmetadata(output).schema, expected) + end + io = IOBuffer() + Parquet.write(io, table; pageversion=:v2, codec=:zstd) + @test wpschemaequal(wpmetadata(take!(io)).schema, expected) +end + +@testset "schema-bearing writer preserves file key-value metadata" begin + bytes = Parquet._encodefile((value=Int32[1],)) + original = wpmetadata(bytes) + keyvalue = WPMD.KeyValue(key="ARROW:schema", value="opaque-schema", + unknown_fields=[wprawi32(90, 900)]) + bytes = wprewritefooter(bytes, wpreplace(original; + key_value_metadata=[keyvalue])) + table = Parquet.Table(bytes) + expected = only(table.metadata.key_value_metadata) + output = try + error = try + Parquet._encodefile(table; + limits=Parquet.Limits(max_string_bytes=5)) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :string_bytes + Parquet._encodefile(table) + finally + close(table) + end + rewritten = wpmetadata(output) + @test rewritten.key_value_metadata !== nothing + actual = only(rewritten.key_value_metadata) + @test actual.key == expected.key + @test actual.value == expected.value + @test Parquet._provenanceexact(actual.unknown_fields, + expected.unknown_fields) +end + +@testset "schema-bearing writer selectors" begin + path = wpfixture("list_columns.parquet") + if isfile(path) + table = Parquet.Table(path) + expected = table.metadata.schema + policies = ( + Dict(("int64_list", "list", "item") => :delta_binary_packed, + ("utf8_list", "list", "item") => :delta_byte_array), + Dict(1 => :plain), + Dict("int64_list" => :plain), + ) + for policy in policies + output = Parquet._encodefile(table; encoding=policy, pageversion=:v2) + @test wpschemaequal(wpmetadata(output).schema, expected) + end + @test_throws ArgumentError Parquet._encodefile(table; + encoding=Dict(("missing", "path") => :plain)) + close(table) + ambiguous = Parquet.Table(wpfixture("nullable.impala.parquet")) + try + @test_throws ArgumentError Parquet._encodefile(ambiguous; + encoding=Dict("nested_struct" => :plain)) + finally + close(ambiguous) + end + else + @info "skipping provenance selector corpus tests" root=WRITE_PROVENANCE_CORPUS + end +end + +@testset "schema-bearing writer corpus layouts" begin + fixtures = ( + "list_columns.parquet", + "null_list.parquet", + "datapage_v2.snappy.parquet", + "old_list_structure.parquet", + "nested_lists.snappy.parquet", + "nested_maps.snappy.parquet", + "repeated_primitive_no_list.parquet", + "repeated_no_annotation.parquet", + "nullable.impala.parquet", + "nonnullable.impala.parquet", + "map_no_value.parquet", + "incorrect_map_schema.parquet", + "nested_structs.rust.parquet", + ) + if all(name -> isfile(wpfixture(name)), fixtures) + for name in fixtures + table = Parquet.Table(wpfixture(name)) + try + wpexactrewrite(table; pageversion=:v1, codec=:uncompressed) + finally + close(table) + end + end + for name in ("list_columns.parquet", "nested_maps.snappy.parquet", + "nullable.impala.parquet") + table = Parquet.Table(wpfixture(name)) + try + wpexactrewrite(table; pageversion=:v2, codec=:zstd) + finally + close(table) + end + end + else + @info "skipping provenance layout corpus tests" root=WRITE_PROVENANCE_CORPUS + end +end + +@testset "schema-bearing writer zero rows" begin + empty = ( + s=NamedTuple{(:id,),Tuple{Int32}}[], + l=Vector{Union{Missing,Int32}}[], + m=Dict{String,Union{Missing,Int32}}[], + ) + table = Parquet.Table(Parquet._encodefile(empty)) + @test table.rows == 0 + for pageversion in (:v1, :v2) + output = wpexactrewrite(table; pageversion=pageversion, + codec=:uncompressed) + metadata = wpmetadata(output) + @test metadata.num_rows == 0 + @test isempty(metadata.row_groups) + end + close(table) +end + +@testset "schema-bearing writer rejects stored-schema tampering" begin + table = wpfreshtable() + changed = copy(table.metadata.schema) + changed[1] = wpreplace(changed[1]; field_id=Int32(777)) + table.metadata = wpreplace(table.metadata; schema=changed) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + changed = copy(table.metadata.schema) + leafindex = findfirst(element -> element.name == "id", changed) + changed[leafindex] = wpreplace(changed[leafindex]; type_=WPMD.Type.INT64) + table.metadata = wpreplace(table.metadata; schema=changed) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + changed = copy(table.metadata.schema) + changed[1] = wpreplace(changed[1]; + num_children=changed[1].num_children + Int32(1)) + table.metadata = wpreplace(table.metadata; schema=changed) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + changed = copy(table.metadata.schema) + changed[1] = wpreplace(changed[1]; field_id=Int32(778)) + changedmetadata = wpreplace(table.metadata; schema=changed) + table.schema = Parquet.Schema(changedmetadata) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = Parquet.Table(wpcustomschemafile()) + changed = copy(table.metadata.schema) + raw = only(changed[1].unknown_fields) + changedraw = WPTH.RawField(raw.id, raw.type, Int16(raw.previd + 1), + raw.headerlength, copy(raw.bytes)) + @test raw == changedraw + @test !Parquet._provenanceexact(raw, changedraw) + changed[1] = wpreplace(changed[1]; unknown_fields=[changedraw]) + table.metadata = wpreplace(table.metadata; schema=changed) + wprejects(() -> Parquet._encodefile(table)) + close(table) +end + +@testset "unknown-field provenance clone charge includes its vector" begin + fields = WPTH.RawField[wprawi32(90, 900), wprawi32(91, 910)] + expected = Parquet._materializedarraybytes(WPTH.RawField, length(fields)) + for field in fields + fieldcharge = Parquet._materializedsum( + Parquet._MATERIALIZED_OBJECT_BYTES, + Parquet._materializedarraybytes(UInt8, length(field.bytes))) + expected = Parquet._materializedsum(expected, fieldcharge) + end + @test Parquet._provenanceclonecharge(fields) == expected +end + +@testset "schema-bearing writer rejects vector tampering" begin + table = wpfreshtable() + table.columns.s.names[1] = "changed" + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + pop!(table.columns.s.children) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.s.children[1] = Int64[1, 2] + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.s.ranks[1] = Int32(1) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.s.ranks[3] = Int32(2) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.l.offsets[1] = Int32(1) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.l.offsets[end] = Int32(1) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.l.offsets[2] = Int32(1) + table.columns.l.offsets[3] = Int32(1) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.m.offsets[1] = Int32(1) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.columns.m.offsets[end] = Int32(1) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + map = table.columns.m + badmap = typeof(map)(copy(map.offsets), copy(map.validity), + WPNullKeyVector(copy(map.keys)), copy(map.values)) + wpreplacecolumn!(table, :m, badmap) + wprejects(() -> Parquet._encodefile(table)) + close(table) + + table = wpfreshtable() + table.rows += 1 + wprejects(() -> Parquet._encodefile(table)) + close(table) +end + +@testset "schema-bearing writer limits and cleanup" begin + values = Union{Missing,NamedTuple{(:text,),Tuple{String}}}[ + (text=repeat("x", 32),), missing] + table = Parquet.Table(Parquet._encodefile((; values))) + limits = Parquet.Limits(max_string_bytes=16) + budget = Parquet._LiveByteBudget(limits) + Parquet._reserve!(budget, 64) + before = Parquet._budgetused(budget) + @test_throws Parquet.LimitError Parquet._provenancewritefields( + table, limits, budget, nothing, false) + @test Parquet._budgetused(budget) == before + Parquet._release!(budget, before) + @test Parquet._budgetused(budget) == 0 + + tight = Parquet.Limits(max_materialized_bytes=512) + tightbudget = Parquet._LiveByteBudget(tight) + @test_throws Parquet.LimitError Parquet._provenancewritefields( + table, tight, tightbudget, nothing, false) + @test Parquet._budgetused(tightbudget) == 0 + @test_throws Parquet.LimitError Parquet._encodefile(table; + limits=Parquet.Limits(max_container_elements=1)) + close(table) +end + +@testset "schema-bearing writer charged name snapshot" begin + short = wpprovenanceallocations(:x) + long = wpprovenanceallocations(Symbol(repeat("x", 4096))) + @test long.bindingbytes <= short.bindingbytes + 512 + @test long.validationbytes <= short.validationbytes + 512 +end + +@testset "schema-bearing writer iterative deep topology" begin + depth = 4096 + for rows in (0, 1) + table, limits = wpdeepstructtable(depth, rows) + try + output = Parquet._encodefile(table; limits=limits) + metadata = wpmetadata(output) + @test metadata.num_rows == rows + @test length(metadata.schema) == depth + 2 + @test metadata.schema[2].name == "deep" + @test metadata.schema[end].name == "value" + error = try + Parquet.Table(output; limits=limits) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :nested_read_depth + finally + close(table) + end + end + + table, limits = wpdeepstructtable(256, 1) + try + output = Parquet._encodefile(table; limits=limits) + rewritten = Parquet.Table(output; limits=limits) + try + value = rewritten.columns.deep[1] + for _ in 1:256 + value = value[1] + end + @test value == Int32(7) + finally + close(rewritten) + end + shallow = Parquet.Limits(max_metadata_depth=257, + max_container_elements=limits.max_container_elements) + budget = Parquet._LiveByteBudget(shallow) + Parquet._reserve!(budget, Int64(64)) + error = try + Parquet._provenancewritefields(table, shallow, budget, nothing, + false) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :metadata_depth + @test Parquet._budgetused(budget) == 64 + finally + close(table) + end +end + +@testset "schema-bearing writer recursive key-only MAP" begin + depth = 256 + table, limits = wpdeepstructkeytable(depth) + try + output = Parquet._encodefile(table; limits=limits) + rewritten = Parquet.Table(output; limits=limits) + try + pair = only(only(rewritten.columns.m)) + @test ismissing(pair.second) + value = pair.first + for _ in 1:depth + value = value[1] + end + @test value == Int32(7) + finally + close(rewritten) + end + finally + close(table) + end +end diff --git a/test/write_splitting.jl b/test/write_splitting.jl new file mode 100644 index 0000000..aecf9ce --- /dev/null +++ b/test/write_splitting.jl @@ -0,0 +1,700 @@ +using Dates +using Test + +const WSMD = Parquet.Metadata +const WSTH = Parquet.Thrift + +function wsframes(file::Parquet.File, chunk::WSMD.ColumnChunk) + metadata = something(chunk.meta_data) + start = metadata.dictionary_page_offset === nothing ? + metadata.data_page_offset : min(metadata.data_page_offset, + metadata.dictionary_page_offset) + stop = Base.checked_add(start, metadata.total_compressed_size) + @test 0 <= start < stop <= file.footer.offset + output = [] + position = start + while position < stop + frame = Parquet.readpage(file.source, position, stop, Parquet.Limits()) + frameend = Parquet.pageend(frame) + @test frameend > position + @test frameend <= stop + push!(output, ( + offset=position, + header=frame.header, + headerlength=frame.headerlength, + payload=collect(frame.payload), + frameend=frameend, + )) + position = frameend + end + @test position == stop + return output +end + +function wsinspect(bytes::Vector{UInt8}) + file = Parquet.File(bytes) + try + metadata = WSTH.decode(copy(file.footer.bytes), WSMD.FileMetaData) + schema = Parquet.Schema(metadata.schema) + groups = [] + for group in metadata.row_groups + push!(groups, [wsframes(file, chunk) for chunk in group.columns]) + end + return (; metadata, schema, groups, footer_offset=file.footer.offset) + finally + close(file) + end +end + +function wsmetadata(bytes::Vector{UInt8}) + file = Parquet.File(bytes) + try + return WSTH.decode(copy(file.footer.bytes), WSMD.FileMetaData) + finally + close(file) + end +end + +function wsdatapage(frame) + type = frame.header.type_ + return type == WSMD.PageType.DATA_PAGE || + type == WSMD.PageType.DATA_PAGE_V2 +end + +function wsframeencoding(frame) + header = frame.header + header.type_ == WSMD.PageType.DICTIONARY_PAGE && + return header.dictionary_page_header.encoding + header.type_ == WSMD.PageType.DATA_PAGE && + return header.data_page_header.encoding + header.type_ == WSMD.PageType.DATA_PAGE_V2 && + return header.data_page_header_v2.encoding + throw(ArgumentError("unexpected writer page type $(header.type_)")) +end + +function wsframevalues(frame) + header = frame.header + header.type_ == WSMD.PageType.DATA_PAGE && + return Int64(header.data_page_header.num_values) + header.type_ == WSMD.PageType.DATA_PAGE_V2 && + return Int64(header.data_page_header_v2.num_values) + return Int64(0) +end + +function wsderivedstats(frames) + keys = Tuple{WSMD.PageType.T,WSMD.Encoding.T}[] + counts = Int32[] + for frame in frames + key = (frame.header.type_, wsframeencoding(frame)) + index = findfirst(==(key), keys) + if index === nothing + push!(keys, key) + push!(counts, Int32(1)) + else + counts[index] = Base.checked_add(counts[index], Int32(1)) + end + end + return WSMD.PageEncodingStats[ + WSMD.PageEncodingStats(page_type=key[1], encoding=key[2], + count=count) for (key, count) in zip(keys, counts) + ] +end + +function wscheckaccounting(result) + metadata = result.metadata + for (groupindex, (group, chunks)) in enumerate(zip(metadata.row_groups, + result.groups)) + compressed = Int64(0) + uncompressed = Int64(0) + for (chunk, frames) in zip(group.columns, chunks) + column = something(chunk.meta_data) + @test chunk.file_offset == 0 + framecompressed = sum(frame -> frame.frameend - frame.offset, + frames; init=Int64(0)) + frameuncompressed = sum(frame -> Int64(frame.headerlength) + + Int64(frame.header.uncompressed_page_size), frames; + init=Int64(0)) + @test column.total_compressed_size == framecompressed + @test column.total_uncompressed_size == frameuncompressed + @test column.num_values == sum(wsframevalues, frames; + init=Int64(0)) + data = filter(wsdatapage, frames) + @test !isempty(data) + @test column.data_page_offset == first(data).offset + dictionaries = filter(frame -> + frame.header.type_ == WSMD.PageType.DICTIONARY_PAGE, frames) + if isempty(dictionaries) + @test column.dictionary_page_offset === nothing + else + @test length(dictionaries) == 1 + @test column.dictionary_page_offset == only(dictionaries).offset + @test first(frames).offset == only(dictionaries).offset + end + @test column.encoding_stats == wsderivedstats(frames) + compressed = Base.checked_add(compressed, framecompressed) + uncompressed = Base.checked_add(uncompressed, frameuncompressed) + end + @test group.total_compressed_size == compressed + @test group.total_byte_size == uncompressed + @test group.file_offset == first(first(chunks)).offset + @test group.num_rows > 0 + @test groupindex <= length(metadata.row_groups) + end + return +end + +function wsdecodelevels(bytes::AbstractVector{UInt8}, count::Int, + maximum::Int16; offset::Int=1, length_prefix::Bool=false) + iszero(maximum) && return zeros(UInt64, count), offset + return Parquet.decode_hybrid(bytes, count, + Parquet._levelbitwidth(maximum); offset=offset, + length_prefix=length_prefix) +end + +function wslevels(frame, node::Parquet.SchemaNode, + codec::WSMD.CompressionCodec.T) + header = frame.header + if header.type_ == WSMD.PageType.DATA_PAGE + count = Int(header.data_page_header.num_values) + bytes = Parquet.decompress(codec, frame.payload, + header.uncompressed_page_size) + repetition, position = wsdecodelevels(bytes, count, + node.max_repetition_level; length_prefix=true) + definition, position = wsdecodelevels(bytes, count, + node.max_definition_level; offset=position, length_prefix=true) + return (; repetition, definition, + values=collect(@view bytes[position:end])) + end + @test header.type_ == WSMD.PageType.DATA_PAGE_V2 + page = header.data_page_header_v2 + count = Int(page.num_values) + repetitionlength = Int(page.repetition_levels_byte_length) + definitionlength = Int(page.definition_levels_byte_length) + definitionstart = repetitionlength + 1 + definitionstop = repetitionlength + definitionlength + repetitionbytes = @view frame.payload[1:repetitionlength] + definitionbytes = @view frame.payload[definitionstart:definitionstop] + repetition, repetitionposition = wsdecodelevels(repetitionbytes, count, + node.max_repetition_level) + definition, definitionposition = wsdecodelevels(definitionbytes, count, + node.max_definition_level) + @test repetitionposition == length(repetitionbytes) + 1 + @test definitionposition == length(definitionbytes) + 1 + encoded = @view frame.payload[(definitionstop + 1):end] + expected = Int(header.uncompressed_page_size) - repetitionlength - + definitionlength + values = if something(page.is_compressed, true) + Parquet.decompress(codec, encoded, expected) + else + @test length(encoded) == expected + collect(encoded) + end + return (; repetition, definition, values) +end + +function wsroundtrip(bytes::Vector{UInt8}, expected::NamedTuple) + table = Parquet.Table(bytes) + try + for name in keys(expected) + @test isequal(getproperty(table.columns, name), + getproperty(expected, name)) + end + finally + close(table) + end + return +end + +function wsgoldeninput() + E = Union{Missing,Date} + days = Union{Missing,Vector{E}}[ + missing, + E[], + E[missing], + E[Date(1970, 1, 1), missing, Date(1969, 12, 31)], + E[Date(2000, 2, 29)], + ] + return (; id=Int32[1, 2, 3, 4, 5], days) +end + +function wsdatacounts(frames) + return Int32[frame.header.type_ == WSMD.PageType.DATA_PAGE ? + frame.header.data_page_header.num_values : + frame.header.data_page_header_v2.num_values + for frame in frames if wsdatapage(frame)] +end + +function wsdecodedictionary(frame, codec::WSMD.CompressionCodec.T) + header = frame.header + @test header.type_ == WSMD.PageType.DICTIONARY_PAGE + count = Int(header.dictionary_page_header.num_values) + bytes = Parquet.decompress(codec, frame.payload, + header.uncompressed_page_size) + values, position = Parquet.decode_plain_byte_array(bytes, count) + @test position == length(bytes) + 1 + return String[String(value) for value in values] +end + +@testset "writer row-boundary prefixes" begin + input = wsgoldeninput() + plan = Parquet._writeplan(input) + leaf = only(filter(leaf -> leaf.path == ["days", "list", "element"], + only(plan.rowgroups).leaves)) + @test leaf.entry_offsets == Int64[0, 1, 2, 3, 6, 7] + @test leaf.dense_offsets == Int64[0, 0, 0, 0, 2, 3] + @test leaf.payload_offsets == Int64[0, 0, 0, 0, 8, 12] + @test leaf.column.repetitions == UInt64[0, 0, 0, 0, 1, 1, 0] + @test leaf.column.definitions == UInt64[0, 1, 2, 3, 2, 3, 3] + @test leaf.column.values == Int32[0, -1, 11016] + for prefix in (leaf.entry_offsets, leaf.dense_offsets, + leaf.payload_offsets) + @test length(prefix) == length(input.id) + 1 + @test first(prefix) == 0 + @test issorted(prefix) + end +end + +@testset "writer row groups and nested slices" begin + input = wsgoldeninput() + expectedrepetition = ( + UInt64[0, 0], UInt64[0, 0, 1, 1], UInt64[0]) + expecteddefinition = ( + UInt64[0, 1], UInt64[2, 3, 2, 3], UInt64[3]) + expectedphysical = (Int32[], Int32[0, -1], Int32[11016]) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; rowgroupsize=2, + pagesize=nothing, pageversion=pageversion) + result = wsinspect(bytes) + @test result.metadata.num_rows == 5 + @test [group.num_rows for group in result.metadata.row_groups] == + Int64[2, 2, 1] + @test [group.ordinal for group in result.metadata.row_groups] == + Union{Nothing,Int16}[0, 1, 2] + @test [group.columns[1].meta_data.num_values + for group in result.metadata.row_groups] == Int64[2, 2, 1] + @test [group.columns[2].meta_data.num_values + for group in result.metadata.row_groups] == Int64[2, 4, 1] + @test all(group -> [chunk.meta_data.path_in_schema + for chunk in group.columns] == [["id"], + ["days", "list", "element"]], result.metadata.row_groups) + for groupindex in 1:3 + frame = only(filter(wsdatapage, result.groups[groupindex][2])) + levels = wslevels(frame, result.schema.leaves[2], + WSMD.CompressionCodec.UNCOMPRESSED) + @test levels.repetition == expectedrepetition[groupindex] + @test levels.definition == expecteddefinition[groupindex] + physical, position = Parquet.decode_plain(Int32, levels.values, + length(expectedphysical[groupindex])) + @test physical == expectedphysical[groupindex] + @test position == length(levels.values) + 1 + if pageversion === :v2 + page = frame.header.data_page_header_v2 + @test page.num_rows == Int32[2, 2, 1][groupindex] + @test page.num_nulls == Int32[2, 2, 0][groupindex] + end + end + wscheckaccounting(result) + wsroundtrip(bytes, input) + end + one = Parquet._encodefile(input; rowgroupsize=nothing, + pagesize=nothing) + @test length(wsinspect(one).metadata.row_groups) == 1 + @test Parquet._encodefile(input) == Parquet._encodefile(input; + rowgroupsize=1_048_576, pagesize=1024 * 1024) +end + +@testset "writer soft page targets" begin + input = (; value=Int32[1, 2, 3, 4, 5]) + for pageversion in (:v1, :v2) + for (pagesize, expected) in ((nothing, Int32[5]), + (8, Int32[2, 2, 1]), (4, Int32[1, 1, 1, 1, 1])) + bytes = Parquet._encodefile(input; rowgroupsize=nothing, + pagesize=pagesize, pageversion=pageversion) + result = wsinspect(bytes) + @test wsdatacounts(only(result.groups)[1]) == expected + @test all(frame -> begin + levels = wslevels(frame, only(result.schema.leaves), + WSMD.CompressionCodec.UNCOMPRESSED) + !isempty(levels.repetition) && iszero(first(levels.repetition)) + end, filter(wsdatapage, only(result.groups)[1])) + wscheckaccounting(result) + wsroundtrip(bytes, input) + end + end +end + +@testset "writer hard page retry and propagation" begin + input = (; value=Int32.(1:16)) + limits = Parquet.Limits(max_page_bytes=16) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; pagesize=nothing, + pageversion=pageversion, limits=limits) + result = wsinspect(bytes) + @test wsdatacounts(only(result.groups)[1]) == Int32[4, 4, 4, 4] + @test all(frame -> frame.header.compressed_page_size <= 16 && + frame.header.uncompressed_page_size <= 16, + filter(wsdatapage, only(result.groups)[1])) + wscheckaccounting(result) + wsroundtrip(bytes, input) + end + + plan = Parquet._writeplan(input) + leaf = only(only(plan.rowgroups).leaves) + budget = Parquet._LiveByteBudget(limits) + pages, charge = Parquet._budgetedsplitcolumnpages(leaf, limits, budget; + pagesize=nothing, checksum=true, dictionary=false, + codec=WSMD.CompressionCodec.UNCOMPRESSED, compressionlevel=nothing, + pageversion=:v1) + @test only(pages.encoding_stats).count == 4 + @test Parquet._budgetused(budget) == charge + Parquet._release!(budget, charge) + @test Parquet._budgetused(budget) == 0 + + # A page build that throws must release its working reservation, leaving only + # what the caller already held. + failbudget = Parquet._LiveByteBudget(Parquet.Limits()) + Parquet._reserve!(failbudget, Int64(64)) + failerror = try + Parquet._budgetedsplitcolumnpages(leaf, + Parquet.Limits(max_page_bytes=3), failbudget; + pagesize=nothing, checksum=true, dictionary=false, + codec=WSMD.CompressionCodec.UNCOMPRESSED, compressionlevel=nothing, + pageversion=:v1) + nothing + catch err + err + end + @test failerror isa Parquet.LimitError + @test Parquet._budgetused(failbudget) == 64 + + error = try + Parquet._encodefile((value=Int32[1],); pagesize=nothing, + limits=Parquet.Limits(max_page_bytes=3)) + nothing + catch err + err + end + @test error isa Parquet.LimitError + @test error.resource == :page_bytes + headererror = try + Parquet._encodefile(input; pagesize=nothing, + limits=Parquet.Limits(max_page_header_bytes=1)) + nothing + catch err + err + end + @test headererror isa Parquet.LimitError + @test headererror.resource == :page_header_bytes + stringerror = try + Parquet._encodefile((value=["large"],); pagesize=nothing, + limits=Parquet.Limits(max_string_bytes=4)) + nothing + catch err + err + end + @test stringerror isa Parquet.LimitError + @test stringerror.resource == :string_bytes + @test_throws ArgumentError Parquet._encodefile(input; pagesize=nothing, + encoding=:delta_byte_array) + io = IOBuffer() + @test_throws Parquet.LimitError Parquet.write(io, (value=Int32[1],); + pagesize=nothing, limits=Parquet.Limits(max_page_bytes=3)) + @test isempty(take!(io)) + mktempdir() do directory + path = joinpath(directory, "failed.parquet") + @test_throws Parquet.LimitError Parquet.write(path, + (value=Int32[1],); pagesize=nothing, + limits=Parquet.Limits(max_page_bytes=3)) + @test !ispath(path) + end +end + +@testset "writer dictionary scope and fallback" begin + reset = (; value=vcat(fill("alpha", 128), fill("beta", 128))) + bytes = Parquet._encodefile(reset; rowgroupsize=128, + pagesize=nothing, dictionary=true) + result = wsinspect(bytes) + @test length(result.metadata.row_groups) == 2 + for (index, expected) in enumerate(("alpha", "beta")) + frames = result.groups[index][1] + @test [frame.header.type_ for frame in frames] == + [WSMD.PageType.DICTIONARY_PAGE, WSMD.PageType.DATA_PAGE] + @test wsdecodedictionary(first(frames), + WSMD.CompressionCodec.UNCOMPRESSED) == [expected] + @test result.metadata.row_groups[index].columns[1].meta_data.encoding_stats == + WSMD.PageEncodingStats[ + WSMD.PageEncodingStats(page_type=WSMD.PageType.DICTIONARY_PAGE, + encoding=WSMD.Encoding.PLAIN, count=Int32(1)), + WSMD.PageEncodingStats(page_type=WSMD.PageType.DATA_PAGE, + encoding=WSMD.Encoding.RLE_DICTIONARY, count=Int32(1)), + ] + end + wscheckaccounting(result) + wsroundtrip(bytes, reset) + + mixed = (; value=vcat(fill("same", 128), + ["unique-$(lpad(index, 4, '0'))" for index in 1:128])) + for policy in ((dictionary=true,), (encoding=:dictionary,)) + bytes = Parquet._encodefile(mixed; rowgroupsize=128, + pagesize=nothing, policy...) + result = wsinspect(bytes) + firstcolumn = result.metadata.row_groups[1].columns[1].meta_data + secondcolumn = result.metadata.row_groups[2].columns[1].meta_data + @test firstcolumn.dictionary_page_offset !== nothing + @test secondcolumn.dictionary_page_offset === nothing + @test firstcolumn.encoding_stats == WSMD.PageEncodingStats[ + WSMD.PageEncodingStats(page_type=WSMD.PageType.DICTIONARY_PAGE, + encoding=WSMD.Encoding.PLAIN, count=Int32(1)), + WSMD.PageEncodingStats(page_type=WSMD.PageType.DATA_PAGE, + encoding=WSMD.Encoding.RLE_DICTIONARY, count=Int32(1)), + ] + @test secondcolumn.encoding_stats == WSMD.PageEncodingStats[ + WSMD.PageEncodingStats(page_type=WSMD.PageType.DATA_PAGE, + encoding=WSMD.Encoding.PLAIN, count=Int32(1)), + ] + wscheckaccounting(result) + wsroundtrip(bytes, mixed) + end + + split = (; value=fill("same", 128)) + bytes = Parquet._encodefile(split; dictionary=true, pagesize=16) + result = wsinspect(bytes) + frames = only(result.groups)[1] + @test first(frames).header.type_ == WSMD.PageType.DICTIONARY_PAGE + @test count(wsdatapage, frames) == 64 + @test count(frame -> frame.header.type_ == WSMD.PageType.DICTIONARY_PAGE, + frames) == 1 + @test result.metadata.row_groups[1].columns[1].meta_data.encoding_stats == + WSMD.PageEncodingStats[ + WSMD.PageEncodingStats(page_type=WSMD.PageType.DICTIONARY_PAGE, + encoding=WSMD.Encoding.PLAIN, count=Int32(1)), + WSMD.PageEncodingStats(page_type=WSMD.PageType.DATA_PAGE, + encoding=WSMD.Encoding.RLE_DICTIONARY, count=Int32(64)), + ] + wscheckaccounting(result) + wsroundtrip(bytes, split) + + tie = (; value=fill(Int32(1), 5)) + plan = Parquet._writeplan(tie) + leaf = only(only(plan.rowgroups).leaves) + limits = Parquet.Limits() + plain = Parquet._writeencodedchunk(leaf, nothing, WSMD.Encoding.PLAIN, + limits; checksum=false, codec=WSMD.CompressionCodec.UNCOMPRESSED, + compressionlevel=nothing, pageversion=:v1) + dictionary = Parquet._writedictionarychunk(leaf, nothing, limits; + checksum=false, codec=WSMD.CompressionCodec.UNCOMPRESSED, + compressionlevel=nothing, pageversion=:v1) + @test length(plain.bytes) == length(dictionary.bytes) + result = wsinspect(Parquet._encodefile(tie; dictionary=true, + checksum=false, pagesize=nothing)) + @test result.metadata.row_groups[1].columns[1].meta_data.dictionary_page_offset === + nothing + + rescue = (; value=[fill("x", 20)]) + rescuelimits = Parquet.Limits(max_page_bytes=20) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(rescue; dictionary=true, checksum=false, + pagesize=nothing, pageversion=pageversion, limits=rescuelimits) + result = wsinspect(bytes) + frames = only(result.groups)[1] + datapagetype = pageversion === :v1 ? WSMD.PageType.DATA_PAGE : + WSMD.PageType.DATA_PAGE_V2 + @test [frame.header.type_ for frame in frames] == + [WSMD.PageType.DICTIONARY_PAGE, datapagetype] + column = result.metadata.row_groups[1].columns[1].meta_data + @test column.dictionary_page_offset == first(frames).offset + @test column.data_page_offset == last(frames).offset + @test wsframeencoding(last(frames)) == WSMD.Encoding.RLE_DICTIONARY + wscheckaccounting(result) + wsroundtrip(bytes, rescue) + end +end + +@testset "writer V1 V2 codec splitting matrix" begin + input = wsgoldeninput() + codecs = ( + (:uncompressed, WSMD.CompressionCodec.UNCOMPRESSED), + (:snappy, WSMD.CompressionCodec.SNAPPY), + (:gzip, WSMD.CompressionCodec.GZIP), + (:brotli, WSMD.CompressionCodec.BROTLI), + (:zstd, WSMD.CompressionCodec.ZSTD), + (:lz4_raw, WSMD.CompressionCodec.LZ4_RAW), + ) + expectedrepetition = ( + UInt64[0], UInt64[0], UInt64[0], UInt64[0, 1, 1], UInt64[0]) + expecteddefinition = ( + UInt64[0], UInt64[1], UInt64[2], UInt64[3, 2, 3], UInt64[3]) + expectedphysical = (Int32[], Int32[], Int32[], Int32[0, -1], + Int32[11016]) + for pageversion in (:v1, :v2), (codecname, codec) in codecs + bytes = Parquet._encodefile(input; rowgroupsize=2, pagesize=1, + pageversion=pageversion, codec=codecname) + result = wsinspect(bytes) + @test [group.num_rows for group in result.metadata.row_groups] == + Int64[2, 2, 1] + @test all(group -> all(chunk -> chunk.meta_data.codec == codec, + group.columns), result.metadata.row_groups) + dayframes = [frame for group in result.groups + for frame in group[2] if wsdatapage(frame)] + @test wsdatacounts(dayframes) == Int32[1, 1, 1, 3, 1] + for index in 1:5 + levels = wslevels(dayframes[index], result.schema.leaves[2], codec) + @test levels.repetition == expectedrepetition[index] + @test levels.definition == expecteddefinition[index] + @test iszero(first(levels.repetition)) + physical, position = Parquet.decode_plain(Int32, levels.values, + length(expectedphysical[index])) + @test physical == expectedphysical[index] + @test position == length(levels.values) + 1 + if pageversion === :v2 + page = dayframes[index].header.data_page_header_v2 + @test page.num_rows == 1 + @test page.num_nulls == Int32[1, 1, 1, 1, 0][index] + end + end + wscheckaccounting(result) + wsroundtrip(bytes, input) + end +end + +@testset "writer Boolean row and page slices" begin + values = Bool[true, false, true, true, false, false, true, false, + true, true, false, true, false, true, true, false, false, true, true] + input = (; value=values) + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile(input; rowgroupsize=nothing, + pagesize=1, pageversion=pageversion, dictionary=true) + result = wsinspect(bytes) + frames = filter(wsdatapage, only(result.groups)[1]) + @test wsdatacounts(frames) == Int32[8, 8, 3] + decoded = Bool[] + for frame in frames + levels = wslevels(frame, only(result.schema.leaves), + WSMD.CompressionCodec.UNCOMPRESSED) + count = length(levels.definition) + valuespart, position = Parquet.decode_plain(Bool, levels.values, + count) + append!(decoded, valuespart) + @test position == length(levels.values) + 1 + end + @test decoded == values + @test result.metadata.row_groups[1].columns[1].meta_data.dictionary_page_offset === + nothing + wscheckaccounting(result) + wsroundtrip(bytes, input) + + grouped = Parquet._encodefile(input; rowgroupsize=5, + pagesize=nothing, pageversion=pageversion) + groupedresult = wsinspect(grouped) + @test [group.num_rows for group in groupedresult.metadata.row_groups] == + Int64[5, 5, 5, 4] + @test [only(wsdatacounts(group[1])) for group in groupedresult.groups] == + Int32[5, 5, 5, 4] + wscheckaccounting(groupedresult) + wsroundtrip(grouped, input) + end +end + +@testset "writer zero rows and split keyword validation" begin + empty = ( + scalar=Int32[], + list=Vector{Union{Missing,Int32}}[], + map=Dict{String,Union{Missing,Int32}}[], + record=NamedTuple{(:id,),Tuple{Int32}}[], + ) + for pageversion in (:v1, :v2), codec in (:uncompressed, :zstd) + bytes = Parquet._encodefile(empty; pageversion=pageversion, + codec=codec, rowgroupsize=1, pagesize=1, dictionary=true) + result = wsinspect(bytes) + @test result.metadata.num_rows == 0 + @test isempty(result.metadata.row_groups) + @test isempty(result.groups) + wsroundtrip(bytes, empty) + end + + input = (; value=Int32[1, 2]) + for invalid in (0, -1, true, 1.0, "1", :invalid) + @test_throws ArgumentError Parquet._encodefile(input; + rowgroupsize=invalid) + @test_throws ArgumentError Parquet._encodefile(input; + pagesize=invalid) + end + @test_throws ArgumentError Parquet._encodefile(input; + rowgroupsize=typemax(UInt128)) + @test_throws ArgumentError Parquet._encodefile(input; + pagesize=typemax(UInt128)) + @test length(wsinspect(Parquet._encodefile(input; + rowgroupsize=nothing, pagesize=nothing)).metadata.row_groups) == 1 + io = IOBuffer() + @test_throws ArgumentError Parquet.write(io, input; pagesize=0) + @test isempty(take!(io)) +end + +@testset "writer omits overflowing row-group ordinals" begin + rows = Int(typemax(Int16)) + 2 + bytes = Parquet._encodefile((value=fill(true, rows),); + rowgroupsize=1, pagesize=nothing, checksum=false) + metadata = wsmetadata(bytes) + @test length(metadata.row_groups) == rows + @test all(group -> group.ordinal === nothing, metadata.row_groups) +end + +@testset "writer plan construction budget rollback" begin + limits = Parquet.Limits() + field = Parquet._writefieldplan(Parquet._writecolumn(:value, Int32[1])) + ordinary = Parquet._LiveByteBudget(limits) + Parquet._reservearray!(ordinary, UInt8, 0) + ordinaryentry = Parquet._budgetused(ordinary) + @test ordinaryentry == 64 + @test_throws ArgumentError Parquet._writeplan( + Parquet.WriteFieldPlan[field], 2, limits, ordinary) + @test Parquet._budgetused(ordinary) == ordinaryentry + + table = Parquet.Table(Parquet._encodefile((value=Int32[1],))) + try + owner = Parquet._LiveByteBudget(limits) + fields, rows = Parquet._writefieldsencoded(table, limits, owner, + nothing, false) + provenance = Parquet._LiveByteBudget(limits) + Parquet._reservearray!(provenance, UInt8, 0) + provenanceentry = Parquet._budgetused(provenance) + @test provenanceentry == 64 + @test_throws ArgumentError Parquet._writeplan(fields, rows + 1, + limits, provenance) + @test Parquet._budgetused(provenance) == provenanceentry + finally + close(table) + end + + columnlimits = Parquet.Limits(max_materialized_bytes=1800) + columnbudget = Parquet._LiveByteBudget(columnlimits) + Parquet._reservearray!(columnbudget, UInt8, 0) + columnentry = Parquet._budgetused(columnbudget) + columns = Parquet.WriteColumn[ + Parquet._writecolumn(:value, Int32[1]), + ] + @test_throws Parquet.LimitError Parquet._writeplan(columns, 1, + columnlimits, columnbudget) + @test Parquet._budgetused(columnbudget) == columnentry == 64 + + ordinarylimits = Parquet.Limits(max_materialized_bytes=5000) + ordinarytable = Parquet._LiveByteBudget(ordinarylimits) + Parquet._reservearray!(ordinarytable, UInt8, 0) + ordinarytableentry = Parquet._budgetused(ordinarytable) + @test_throws Parquet.LimitError Parquet._writeplan( + (value=Int32[1],), ordinarylimits, ordinarytable) + @test Parquet._budgetused(ordinarytable) == ordinarytableentry == 64 + + source = Parquet.Table(Parquet._encodefile((value=Int32[1],))) + try + provenancelimits = Parquet.Limits(max_materialized_bytes=4000) + provenancetable = Parquet._LiveByteBudget(provenancelimits) + Parquet._reservearray!(provenancetable, UInt8, 0) + provenanceentry = Parquet._budgetused(provenancetable) + @test_throws Parquet.LimitError Parquet._writeplan(source, + provenancelimits, provenancetable) + @test Parquet._budgetused(provenancetable) == provenanceentry == 64 + finally + close(source) + end +end diff --git a/test/write_statistics.jl b/test/write_statistics.jl new file mode 100644 index 0000000..e8c5f5b --- /dev/null +++ b/test/write_statistics.jl @@ -0,0 +1,480 @@ +using Dates +using SHA +using UUIDs +import Tables + +const WSMD = Parquet.Metadata + +function wsmetadata(bytes::Vector{UInt8}) + file = Parquet.File(bytes) + return try + Parquet.Thrift.decode(file.footer.bytes, WSMD.FileMetaData) + finally + close(file) + end +end + +function wsstats(metadata::WSMD.FileMetaData, group::Int, column::Int) + return metadata.row_groups[group].columns[column].meta_data.statistics +end + +function wspages(bytes::Vector{UInt8}, column::Int) + file = Parquet.File(bytes) + return try + metadata = Parquet.Thrift.decode(file.footer.bytes, WSMD.FileMetaData) + chunk = metadata.row_groups[1].columns[column].meta_data + start, stop = Parquet._chunkrange(chunk, file.footer.offset) + headers = WSMD.PageHeader[] + position = start + while position < stop + frame = Parquet.readpage(file.source, position, stop, Parquet.Limits()) + push!(headers, frame.header) + position = Parquet.pageend(frame) + end + headers + finally + close(file) + end +end + +function wscolumnindex(metadata::WSMD.FileMetaData, name::String) + for (index, leaf) in enumerate(metadata.schema[2:end]) + leaf.name == name && return index + end + throw(ArgumentError("writer statistics test column $name is absent")) +end + +function wsorderkind(order::WSMD.ColumnOrder) + order.TYPE_ORDER !== nothing && return :type + order.IEEE_754_TOTAL_ORDER !== nothing && return :ieee + return :unknown +end + +function wsle16(bytes::AbstractVector{UInt8}) + length(bytes) == 2 || throw(ArgumentError("expected two bytes")) + return UInt16(bytes[1]) | (UInt16(bytes[2]) << 8) +end + +function wsle32(bytes::AbstractVector{UInt8}) + length(bytes) == 4 || throw(ArgumentError("expected four bytes")) + return UInt32(bytes[1]) | (UInt32(bytes[2]) << 8) | + (UInt32(bytes[3]) << 16) | (UInt32(bytes[4]) << 24) +end + +function wsle64(bytes::AbstractVector{UInt8}) + length(bytes) == 8 || throw(ArgumentError("expected eight bytes")) + value = UInt64(0) + for index in 8:-1:1 + value = (value << 8) | UInt64(bytes[index]) + end + return value +end + +function wsfloatvalue(::Type{Float16}, bits::UInt16) + return reinterpret(Float16, bits) +end + +function wsfloatvalue(::Type{Float32}, bits::UInt32) + return reinterpret(Float32, bits) +end + +function wsfloatvalue(::Type{Float64}, bits::UInt64) + return reinterpret(Float64, bits) +end + +function wsfloatbits(::Type{Float16}, bytes::AbstractVector{UInt8}) + return wsle16(bytes) +end + +function wsfloatbits(::Type{Float32}, bytes::AbstractVector{UInt8}) + return wsle32(bytes) +end + +function wsfloatbits(::Type{Float64}, bytes::AbstractVector{UInt8}) + return wsle64(bytes) +end + +function wsieeekey(bits::T) where {T<:Union{UInt16,UInt32,UInt64}} + sign = one(T) << (8 * sizeof(T) - 1) + return iszero(bits & sign) ? bits | sign : ~bits +end + +function wsassertmodern(statistics::WSMD.Statistics) + @test statistics.min === nothing + @test statistics.max === nothing + if statistics.min_value === nothing + @test statistics.max_value === nothing + @test statistics.is_min_value_exact === nothing + @test statistics.is_max_value_exact === nothing + else + @test statistics.max_value !== nothing + @test statistics.is_min_value_exact === true + @test statistics.is_max_value_exact === true + end + return +end + +mutable struct WSCountingVector{T} <: AbstractVector{T} + values::Vector{T} + reads::Int +end + +function Base.IndexStyle(::Type{<:WSCountingVector}) + return IndexLinear() +end + +function Base.size(values::WSCountingVector) + return size(values.values) +end + +function Base.getindex(values::WSCountingVector, index::Int) + values.reads += 1 + return values.values[index] +end + +mutable struct WSCallbackTable + calls::Int + values::Vector{Int32} +end + +function Tables.istable(::Type{WSCallbackTable}) + return true +end + +function Tables.columnaccess(::Type{WSCallbackTable}) + return true +end + +function Tables.columns(table::WSCallbackTable) + table.calls += 1 + return (value=table.values,) +end + +@testset "writer statistics column orders and logical families" begin + input = ( + flag=Bool[true, false], + i32=Int32[-2, 3], + i64=Int64[-9, 11], + u32=UInt32[typemax(UInt32), 0], + u64=UInt64[typemax(UInt64), 0], + f32=Float32[-0.0, 0.0], + f64=Float64[-Inf, Inf], + text=["a\0b", "a\0c"], + raw=Vector{UInt8}[UInt8[0xff], UInt8[0x00, 0xff]], + fixed=NTuple{3,UInt8}[(0x01, 0x00, 0xff), (0x01, 0x01, 0x00)], + date=Date[Date(1969, 12, 31), Date(2000, 2, 29)], + time=Time[Time(0), Time(23, 59, 59, 999, 999, 999)], + timestamp=Parquet.Timestamp{:nanos}[ + Parquet.Timestamp(-1, :nanos, false), + Parquet.Timestamp(1, :nanos, false), + ], + decimal32=Parquet.Decimal[ + Parquet.Decimal(-12, 1), Parquet.Decimal(34, 1)], + decimal64=Parquet.Decimal[ + Parquet.Decimal(-12_345_678_901, 1), + Parquet.Decimal(12_345_678_902, 1), + ], + decimalfixed=Parquet.Decimal[ + Parquet.Decimal(big"-12345678901234567890", 1), + Parquet.Decimal(big"12345678901234567891", 1), + ], + uuid=UUID[ + UUID("00112233-4455-6677-8899-aabbccddeeff"), + UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"), + ], + f16=Float16[-0.0, 0.0], + json=Parquet.JSONValue[ + Parquet.JSONValue(codeunits("{\"a\":1}")), + Parquet.JSONValue(codeunits("[1,null,3]")), + ], + bson=Parquet.BSONValue[ + Parquet.BSONValue(hex2bytes("0c0000001061000100000000")), + Parquet.BSONValue(hex2bytes("090000000a00ff0000")), + ], + interval=Parquet.Interval[ + Parquet.Interval(0, 1, 2), Parquet.Interval(3, 4, 5)], + unknown=Missing[missing, missing], + ) + metadata = wsmetadata(Parquet._encodefile(input; statistics=true)) + @test metadata.column_orders !== nothing + @test length(metadata.column_orders) == length(metadata.schema) - 1 + floating = Set(["f32", "f64", "f16"]) + for (index, element) in enumerate(metadata.schema[2:end]) + expected = element.name in floating ? :ieee : :type + @test wsorderkind(metadata.column_orders[index]) == expected + statistics = wsstats(metadata, 1, index) + @test statistics !== nothing + wsassertmodern(statistics) + @test statistics.null_count == (element.name == "unknown" ? 2 : 0) + if element.name in floating + @test statistics.nan_count == 0 + else + @test statistics.nan_count === nothing + end + if element.name in ("interval", "unknown") + @test statistics.min_value === nothing + @test statistics.max_value === nothing + else + @test statistics.min_value !== nothing + @test statistics.max_value !== nothing + end + chunk = metadata.row_groups[1].columns[index] + @test chunk.column_index_offset === nothing + @test chunk.column_index_length === nothing + end + unsigned32 = wsstats(metadata, 1, wscolumnindex(metadata, "u32")) + @test wsle32(unsigned32.min_value) == UInt32(0) + @test wsle32(unsigned32.max_value) == typemax(UInt32) + unsigned64 = wsstats(metadata, 1, wscolumnindex(metadata, "u64")) + @test wsle64(unsigned64.min_value) == UInt64(0) + @test wsle64(unsigned64.max_value) == typemax(UInt64) + signed = wsstats(metadata, 1, wscolumnindex(metadata, "i32")) + @test reinterpret(Int32, wsle32(signed.min_value)) == Int32(-2) + @test reinterpret(Int32, wsle32(signed.max_value)) == Int32(3) + text = wsstats(metadata, 1, wscolumnindex(metadata, "text")) + @test text.min_value == collect(codeunits("a\0b")) + @test text.max_value == collect(codeunits("a\0c")) + raw = wsstats(metadata, 1, wscolumnindex(metadata, "raw")) + @test raw.min_value == UInt8[0x00, 0xff] + @test raw.max_value == UInt8[0xff] + fixed = wsstats(metadata, 1, wscolumnindex(metadata, "fixed")) + @test fixed.min_value == UInt8[0x01, 0x00, 0xff] + @test fixed.max_value == UInt8[0x01, 0x01, 0x00] +end + +@testset "writer IEEE extrema, NaN counts, and signed zero" begin + formats = ( + (Float16, UInt16, UInt16(0x8000), UInt16(0x0000), + UInt16(0x7e11), UInt16(0xfe22)), + (Float32, UInt32, UInt32(0x80000000), UInt32(0x00000000), + UInt32(0x7fc00011), UInt32(0xffc00022)), + (Float64, UInt64, UInt64(0x8000000000000000), + UInt64(0x0000000000000000), UInt64(0x7ff8000000000011), + UInt64(0xfff8000000000022)), + ) + for (T, _, negativezero, positivezero, positivenan, negativenan) in formats + mixed = Union{Missing,T}[ + wsfloatvalue(T, negativezero), wsfloatvalue(T, positivenan), + missing, wsfloatvalue(T, positivezero), + ] + metadata = wsmetadata(Parquet._encodefile((value=mixed,); + statistics=true)) + statistics = wsstats(metadata, 1, 1) + @test wsorderkind(only(metadata.column_orders)) == :ieee + @test statistics.null_count == 1 + @test statistics.nan_count == 1 + @test wsfloatbits(T, statistics.min_value) == negativezero + @test wsfloatbits(T, statistics.max_value) == positivezero + allnanbits = (negativenan, positivenan) + allnan = T[wsfloatvalue(T, bits) for bits in allnanbits] + allnanmetadata = wsmetadata(Parquet._encodefile((value=allnan,); + statistics=true)) + allnanstatistics = wsstats(allnanmetadata, 1, 1) + expected = sort(collect(allnanbits); by=wsieeekey) + @test allnanstatistics.nan_count == 2 + @test wsfloatbits(T, allnanstatistics.min_value) == first(expected) + @test wsfloatbits(T, allnanstatistics.max_value) == last(expected) + for bits in (positivezero, negativezero) + zero = wsfloatvalue(T, bits) + zerostatistics = wsstats(wsmetadata(Parquet._encodefile( + (value=T[zero],); statistics=true)), 1, 1) + @test zerostatistics.nan_count == 0 + @test wsfloatbits(T, zerostatistics.min_value) == bits + @test wsfloatbits(T, zerostatistics.max_value) == bits + end + end + allhalves = reinterpret(Float16, collect(UInt16(0):typemax(UInt16))) + statistics = wsstats(wsmetadata(Parquet._encodefile( + (value=allhalves,); statistics=true)), 1, 1) + @test statistics.nan_count == 2046 + @test wsle16(statistics.min_value) == UInt16(0xfc00) + @test wsle16(statistics.max_value) == UInt16(0x7c00) +end + +@testset "writer statistics nested row-group slices" begin + E = Union{Missing,Int32} + rows = Union{Missing,Vector{E}}[ + E[Int32(1), missing], + missing, + E[], + E[Int32(5), Int32(-2)], + E[Int32(7)], + ] + metadata = wsmetadata(Parquet._encodefile((items=rows,); + rowgroupsize=2, statistics=true)) + @test length(metadata.row_groups) == 3 + expected = ((2, Int32(1), Int32(1)), + (1, Int32(-2), Int32(5)), (0, Int32(7), Int32(7))) + for (group, (nulls, lower, upper)) in enumerate(expected) + statistics = wsstats(metadata, group, 1) + @test statistics.null_count == nulls + @test reinterpret(Int32, wsle32(statistics.min_value)) == lower + @test reinterpret(Int32, wsle32(statistics.max_value)) == upper + end +end + +@testset "writer statistics empty and all-null states" begin + empty = wsmetadata(Parquet._encodefile((integer=Int32[], + double=Float64[], half=Float16[]); + statistics=true)) + @test isempty(empty.row_groups) + @test length(empty.column_orders) == 3 + @test wsorderkind.(empty.column_orders) == [:type, :ieee, :ieee] + allnull = Union{Missing,Int32}[missing, missing, missing] + statistics = wsstats(wsmetadata(Parquet._encodefile((value=allnull,); + statistics=true)), 1, 1) + @test statistics.null_count == 3 + @test statistics.nan_count === nothing + @test statistics.min_value === nothing + @test statistics.max_value === nothing + wsassertmodern(statistics) +end + +@testset "writer statistics value limit" begin + values = (value=Int32[-1, 2],) + exact = wsstats(wsmetadata(Parquet._encodefile(values; + statistics=true, + limits=Parquet.Limits(max_statistics_value_bytes=4))), 1, 1) + @test exact.min_value !== nothing + @test exact.max_value !== nothing + over = wsstats(wsmetadata(Parquet._encodefile(values; + statistics=true, + limits=Parquet.Limits(max_statistics_value_bytes=3))), 1, 1) + @test over.null_count == 0 + @test over.min_value === nothing + @test over.max_value === nothing + variable = (value=Vector{UInt8}[UInt8[0x01, 0x02], + UInt8[0x01, 0x02, 0x03]],) + variableexact = wsstats(wsmetadata(Parquet._encodefile(variable; + statistics=true, + limits=Parquet.Limits(max_statistics_value_bytes=3))), 1, 1) + @test variableexact.min_value == UInt8[0x01, 0x02] + @test variableexact.max_value == UInt8[0x01, 0x02, 0x03] + variableover = wsstats(wsmetadata(Parquet._encodefile(variable; + statistics=true, + limits=Parquet.Limits(max_statistics_value_bytes=2))), 1, 1) + @test variableover.null_count == 0 + @test variableover.min_value === nothing + @test variableover.max_value === nothing + emptybound = wsstats(wsmetadata(Parquet._encodefile( + (value=Vector{UInt8}[UInt8[]],); statistics=true, + limits=Parquet.Limits(max_statistics_value_bytes=0))), 1, 1) + @test emptybound.min_value == UInt8[] + @test emptybound.max_value == UInt8[] +end + +@testset "writer statistics opt-out and page boundary" begin + input = (id=Int32[1, 2, 3], + label=Union{Missing,String}["a", missing, "b"]) + disabled = Parquet._encodefile(input; statistics=false) + @test bytes2hex(sha256(disabled)) == + "bd0f5655e9f9aca2a1aa5f1d2721d9ea8f5ca3c9a2714385cca49edfcb7c0256" + metadata = wsmetadata(disabled) + @test metadata.column_orders === nothing + @test all(chunk -> chunk.meta_data.statistics === nothing, + metadata.row_groups[1].columns) + @test Parquet._encodefile(input) == Parquet._encodefile(input; + statistics=true) + without = WSCountingVector(Int32[1, 2, 3], 0) + with = WSCountingVector(Int32[1, 2, 3], 0) + Parquet._encodefile((value=without,); statistics=false) + Parquet._encodefile((value=with,); statistics=true) + @test with.reads == without.reads + for pageversion in (:v1, :v2) + bytes = Parquet._encodefile((value=Int32[1, 2, 3],); + pageversion=pageversion, pagesize=4, statistics=true) + headers = wspages(bytes, 1) + @test length(headers) > 1 + for header in headers + statistics = pageversion === :v1 ? + header.data_page_header.statistics : + header.data_page_header_v2.statistics + @test statistics === nothing + end + end +end + +@testset "writer statistics validation and destination atomicity" begin + limits = Parquet.Limits(max_statistics_value_bytes=-1) + table = WSCallbackTable(0, Int32[1]) + io = IOBuffer() + Base.write(io, codeunits("unchanged")) + @test_throws ArgumentError Parquet.write(io, table; limits=limits) + @test table.calls == 0 + @test String(take!(io)) == "unchanged" + table = WSCallbackTable(0, Int32[1]) + mktempdir() do directory + path = joinpath(directory, "existing.parquet") + write(path, "unchanged") + @test_throws ArgumentError Parquet.write(path, table; limits=limits) + @test table.calls == 0 + @test read(path, String) == "unchanged" + absent = joinpath(directory, "absent.parquet") + @test_throws ArgumentError Parquet.write(absent, table; limits=limits) + @test !ispath(absent) + end + @test_throws ArgumentError Parquet._encodefile((value=Int32[1],); + statistics=false, limits=limits) +end + +@testset "writer statistics live-budget rollback" begin + malformed = WSMD.SchemaElement( + type_=WSMD.Type.FIXED_LEN_BYTE_ARRAY, + type_length=Int32(2), + repetition_type=WSMD.FieldRepetitionType.REQUIRED, + name="value", + ) + column = Parquet.WriteColumn("value", Vector{UInt8}[UInt8[0x01]], + WSMD.Type.FIXED_LEN_BYTE_ARRAY, Int32(2), false, nothing, nothing, + String["value"], nothing, nothing, Int16(0), Int16(0), 1, + WSMD.SchemaElement[]) + leaf = Parquet.WriteLeafPlan(Int32(1), String["value"], column) + budget = Parquet._LiveByteBudget(Parquet.Limits()) + before = Parquet._budgetused(budget) + @test_throws ArgumentError Parquet._writecolumnstatistics( + leaf, malformed, Int64(4096), budget) + @test Parquet._budgetused(budget) == before + validcolumn = Parquet._writecolumn("value", Int32[1, 2]) + validelement = only(validcolumn.schema) + validleaf = Parquet.WriteLeafPlan(Int32(1), String["value"], validcolumn) + arraycharge = Parquet._materializedarraybytes(UInt8, 4) + tight = Parquet._LiveByteBudget(Parquet.Limits( + max_materialized_bytes=3 * arraycharge)) + @test_throws Parquet.LimitError Parquet._writecolumnstatistics( + validleaf, validelement, Int64(4096), tight) + @test Parquet._budgetused(tight) == 0 + success = Parquet._LiveByteBudget(Parquet.Limits()) + statistics = Parquet._writecolumnstatistics( + validleaf, validelement, Int64(4096), success) + expected = 2 * Parquet._materializedarraybytes(UInt8, 4) + + Parquet._MATERIALIZED_OBJECT_BYTES + @test statistics.null_count == 0 + @test Parquet._budgetused(success) == expected + Parquet._release!(success, expected) + @test Parquet._budgetused(success) == 0 + elements = WSMD.SchemaElement[ + WSMD.SchemaElement(name="schema", num_children=Int32(1)), + validelement, + ] + schema = Parquet.Schema(elements) + orderarray = Parquet._materializedarraybytes(WSMD.ColumnOrder, 1) + orderbudget = Parquet._LiveByteBudget(Parquet.Limits( + max_materialized_bytes=orderarray)) + @test_throws Parquet.LimitError Parquet._writecolumnorders(schema, orderbudget) + @test Parquet._budgetused(orderbudget) == 0 +end + +@testset "writer fixed statistics allocation stays value-independent" begin + function allocation(values::Vector{Int32}) + column = Parquet._writecolumn("value", values) + leaf = Parquet.WriteLeafPlan(Int32(1), String["value"], column) + budget = Parquet._LiveByteBudget(Parquet.Limits()) + return @allocated Parquet._writecolumnstatistics( + leaf, only(column.schema), Int64(4096), budget) + end + allocation(Int32[1]) + allocation(fill(Int32(1), 100_000)) + small = allocation(Int32[1]) + large = allocation(fill(Int32(1), 100_000)) + @test large <= small + 512 +end diff --git a/thrift/.gitattributes b/thrift/.gitattributes index 36eaad9..1e6aad7 100644 --- a/thrift/.gitattributes +++ b/thrift/.gitattributes @@ -1 +1 @@ -* linguist-vendored +parquet.thrift linguist-vendored diff --git a/thrift/README.md b/thrift/README.md new file mode 100644 index 0000000..26db3f5 --- /dev/null +++ b/thrift/README.md @@ -0,0 +1,27 @@ +# Vendored Parquet IDL + +`parquet.thrift` is the unmodified Apache Parquet format definition from +[apache/parquet-format](https://github.com/apache/parquet-format) release 2.13.0, +peeled source commit `c47e2a66e88943fc46fde1b028a9432f14fdf5c0` +(`src/main/thrift/parquet.thrift`, git blob `fe259d61bc470ade78bad48f5223a82598b91b59`). +It is licensed under the Apache License 2.0; the license header is preserved in the file. + +`generate.jl` is a pure-Julia generator that turns the IDL into +`src/metadata/parquet.jl`, the immutable `Parquet.Metadata` structs decoded and encoded +by the Compact Protocol runtime in `src/thrift.jl`. No external Thrift compiler is used. + +```bash +julia thrift/generate.jl # regenerate src/metadata/parquet.jl +julia thrift/generate.jl --check # fail when the checked-in file is stale +``` + +Generated code conventions: + +- Every struct keeps unrecognized fields (including extension field 32767) in + `unknown_fields::Tuple{Vararg{Thrift.RawField}}` with their verbatim header and payload + bytes, and re-emits them in encounter order. +- Enums are modules with an `Int32` wrapper type `T`; unknown values round-trip unchanged. +- Unions are structs whose members are all optional; decoding and construction reject more + than one member, and a single unknown member is preserved. +- Field names that are Julia keywords get a trailing underscore (`type` becomes `type_`); + the comment next to each field records the Thrift id, requiredness, type, and name. diff --git a/thrift/generate.jl b/thrift/generate.jl new file mode 100644 index 0000000..3ec3f02 --- /dev/null +++ b/thrift/generate.jl @@ -0,0 +1,772 @@ +# Pure-Julia generator for src/metadata/parquet.jl from the vendored Parquet Thrift IDL. +# +# julia thrift/generate.jl # regenerate src/metadata/parquet.jl +# julia thrift/generate.jl --check # exit 1 when the checked-in file is stale +# +# The generator uses no external Thrift compiler. Output is deterministic: it depends +# only on the IDL bytes and this file. +module ThriftGenerator + +const FORMAT_VERSION = "2.13.0" +const FORMAT_COMMIT = "c47e2a66e88943fc46fde1b028a9432f14fdf5c0" +const IDL_PATH = joinpath(@__DIR__, "parquet.thrift") +const OUTPUT_PATH = normpath(joinpath(@__DIR__, "..", "src", "metadata", "parquet.jl")) + +const BASE_TYPES = Dict( + "bool" => "Bool", "byte" => "Int8", "i8" => "Int8", "i16" => "Int16", "i32" => "Int32", + "i64" => "Int64", "double" => "Float64", "string" => "String", "binary" => "Vector{UInt8}") + +const BASE_CODES = Dict( + "byte" => "Thrift.BYTE", "i8" => "Thrift.BYTE", "i16" => "Thrift.I16", "i32" => "Thrift.I32", + "i64" => "Thrift.I64", "double" => "Thrift.DOUBLE", "string" => "Thrift.BINARY", + "binary" => "Thrift.BINARY") + +const BASE_READERS = Dict( + "byte" => "Thrift.readi8(r)", "i8" => "Thrift.readi8(r)", "i16" => "Thrift.readi16(r)", + "i32" => "Thrift.readi32(r)", "i64" => "Thrift.readi64(r)", "double" => "Thrift.readdouble(r)", + "string" => "Thrift.readstring(r)", "binary" => "Thrift.readbinary(r)") + +const BASE_WRITERS = Dict( + "byte" => "Thrift.writei8!", "i8" => "Thrift.writei8!", "i16" => "Thrift.writei16!", + "i32" => "Thrift.writei32!", "i64" => "Thrift.writei64!", "double" => "Thrift.writedouble!", + "string" => "Thrift.writestring!", "binary" => "Thrift.writebinary!") + +# Identifiers that are mangled with a trailing underscore in generated Julia code. +const JULIA_KEYWORDS = Set(["abstract", "baremodule", "begin", "break", "catch", "const", + "continue", "do", "else", "elseif", "end", "export", "false", "finally", "for", "function", + "global", "if", "import", "in", "isa", "let", "local", "macro", "module", "mutable", "primitive", + "quote", "return", "struct", "true", "try", "type", "using", "where", "while"]) + +# --------------------------------------------------------------------------- +# Tokenizer +# --------------------------------------------------------------------------- + +struct Token + kind::Symbol + text::String + line::Int +end + +mutable struct Scanner + text::String + pos::Int + line::Int +end + +function _peekchar(s::Scanner, ahead::Int=0) + pos = s.pos + for _ in 1:ahead + pos > lastindex(s.text) && return '\0' + pos = nextind(s.text, pos) + end + pos > lastindex(s.text) && return '\0' + return s.text[pos] +end + +function _advance!(s::Scanner) + c = s.text[s.pos] + c == '\n' && (s.line += 1) + s.pos = nextind(s.text, s.pos) + return c +end + +function _skipline!(s::Scanner) + while s.pos <= lastindex(s.text) && _peekchar(s) != '\n' + _advance!(s) + end + return +end + +function _skipblockcomment!(s::Scanner) + _advance!(s) + _advance!(s) + while s.pos <= lastindex(s.text) + _peekchar(s) == '*' && _peekchar(s, 1) == '/' && break + _advance!(s) + end + s.pos <= lastindex(s.text) || error("unterminated block comment") + _advance!(s) + _advance!(s) + return +end + +function _isidentstart(c::Char) + return c == '_' || ('a' <= c <= 'z') || ('A' <= c <= 'Z') +end + +function _isidentchar(c::Char) + return _isidentstart(c) || ('0' <= c <= '9') || c == '.' +end + +function _scanwhile!(s::Scanner, predicate) + start = s.pos + while s.pos <= lastindex(s.text) && predicate(_peekchar(s)) + _advance!(s) + end + return s.text[start:prevind(s.text, s.pos)] +end + +function _scanstring!(s::Scanner) + quotechar = _advance!(s) + start = s.pos + while s.pos <= lastindex(s.text) && _peekchar(s) != quotechar + _advance!(s) + end + s.pos <= lastindex(s.text) || error("unterminated string literal") + text = s.text[start:prevind(s.text, s.pos)] + _advance!(s) + return text +end + +function _isnumberchar(c::Char) + return ('0' <= c <= '9') || c == '.' || c == 'x' || ('a' <= c <= 'f') || ('A' <= c <= 'F') +end + +function _scannumber!(s::Scanner) + start = s.pos + _peekchar(s) == '-' && _advance!(s) + _scanwhile!(s, _isnumberchar) + return s.text[start:prevind(s.text, s.pos)] +end + +function tokenize(text::String) + s = Scanner(text, firstindex(text), 1) + tokens = Token[] + while s.pos <= lastindex(s.text) + c = _peekchar(s) + if isspace(c) + _advance!(s) + elseif (c == '/' && _peekchar(s, 1) == '/') || c == '#' + _skipline!(s) + elseif c == '/' && _peekchar(s, 1) == '*' + _skipblockcomment!(s) + elseif _isidentstart(c) + push!(tokens, Token(:ident, _scanwhile!(s, _isidentchar), s.line)) + elseif ('0' <= c <= '9') || (c == '-' && '0' <= _peekchar(s, 1) <= '9') + push!(tokens, Token(:number, _scannumber!(s), s.line)) + elseif c == '"' || c == '\'' + push!(tokens, Token(:string, _scanstring!(s), s.line)) + elseif c in "{}<>():;,=*" + push!(tokens, Token(:punct, string(_advance!(s)), s.line)) + else + error("unexpected character $(repr(c)) on line $(s.line)") + end + end + return tokens +end + +# --------------------------------------------------------------------------- +# Parser +# --------------------------------------------------------------------------- + +struct TypeRef + kind::Symbol + name::String + args::Vector{TypeRef} +end + +struct FieldDef + id::Int + requiredness::Symbol + type::TypeRef + name::String + default::Union{Nothing,Token} +end + +struct EnumDef + name::String + values::Vector{Pair{String,Int32}} +end + +struct StructDef + name::String + kind::Symbol + fields::Vector{FieldDef} +end + +struct TypedefDef + name::String + type::TypeRef +end + +mutable struct Parser + tokens::Vector{Token} + pos::Int +end + +function _peek(p::Parser) + p.pos <= length(p.tokens) && return p.tokens[p.pos] + return Token(:eof, "", 0) +end + +function _next!(p::Parser) + token = _peek(p) + p.pos += 1 + return token +end + +function _expect!(p::Parser, text::String) + token = _next!(p) + token.text == text || error("expected $(repr(text)) but found $(repr(token.text)) on line $(token.line)") + return token +end + +function _expectkind!(p::Parser, kind::Symbol) + token = _next!(p) + token.kind == kind || error("expected $kind but found $(repr(token.text)) on line $(token.line)") + return token +end + +function _accept!(p::Parser, text::String) + _peek(p).text == text || return false + p.pos += 1 + return true +end + +function _skipannotations!(p::Parser) + _accept!(p, "(") || return + depth = 1 + while depth > 0 + token = _next!(p) + token.kind == :eof && error("unterminated annotation") + token.text == "(" && (depth += 1) + token.text == ")" && (depth -= 1) + end + return +end + +function _skipseparator!(p::Parser) + _accept!(p, ",") || _accept!(p, ";") + return +end + +function parsetype!(p::Parser) + token = _expectkind!(p, :ident) + name = token.text + if name == "list" || name == "set" + _expect!(p, "<") + element = parsetype!(p) + _expect!(p, ">") + _accept!(p, "cpp_type") && _expectkind!(p, :string) + return TypeRef(Symbol(name), name, [element]) + elseif name == "map" + _expect!(p, "<") + key = parsetype!(p) + _expect!(p, ",") + value = parsetype!(p) + _expect!(p, ">") + _accept!(p, "cpp_type") && _expectkind!(p, :string) + return TypeRef(:map, name, [key, value]) + end + haskey(BASE_TYPES, name) && return TypeRef(:base, name, TypeRef[]) + return TypeRef(:named, name, TypeRef[]) +end + +function parsefield!(p::Parser) + idtoken = _expectkind!(p, :number) + id = parse(Int, idtoken.text) + _expect!(p, ":") + requiredness = :default + _accept!(p, "required") && (requiredness = :required) + _accept!(p, "optional") && (requiredness = :optional) + type = parsetype!(p) + _skipannotations!(p) + name = _expectkind!(p, :ident).text + default = _accept!(p, "=") ? _next!(p) : nothing + _skipannotations!(p) + _skipseparator!(p) + return FieldDef(id, requiredness, type, name, default) +end + +function parsestruct!(p::Parser, kind::Symbol) + name = _expectkind!(p, :ident).text + _expect!(p, "{") + fields = FieldDef[] + while !_accept!(p, "}") + push!(fields, parsefield!(p)) + end + _skipannotations!(p) + return StructDef(name, kind, fields) +end + +function parseenum!(p::Parser) + name = _expectkind!(p, :ident).text + _expect!(p, "{") + values = Pair{String,Int32}[] + nextvalue = Int32(0) + while !_accept!(p, "}") + entry = _expectkind!(p, :ident).text + value = _accept!(p, "=") ? Int32(parse(Int, _expectkind!(p, :number).text)) : nextvalue + _skipannotations!(p) + _skipseparator!(p) + push!(values, entry => value) + nextvalue = value + Int32(1) + end + _skipannotations!(p) + return EnumDef(name, values) +end + +function parsedocument(tokens::Vector{Token}) + p = Parser(tokens, 1) + definitions = Any[] + while _peek(p).kind != :eof + keyword = _next!(p) + if keyword.text == "namespace" + _next!(p) + _next!(p) + elseif keyword.text == "include" || keyword.text == "cpp_include" + _expectkind!(p, :string) + elseif keyword.text == "enum" + push!(definitions, parseenum!(p)) + elseif keyword.text == "struct" || keyword.text == "union" || keyword.text == "exception" + push!(definitions, parsestruct!(p, keyword.text == "union" ? :union : :struct)) + elseif keyword.text == "typedef" + type = parsetype!(p) + name = _expectkind!(p, :ident).text + _skipseparator!(p) + push!(definitions, TypedefDef(name, type)) + else + error("unsupported Thrift definition $(repr(keyword.text)) on line $(keyword.line)") + end + end + return definitions +end + +# --------------------------------------------------------------------------- +# Emitter +# --------------------------------------------------------------------------- + +struct Context + kinds::Dict{String,Symbol} + typedefs::Dict{String,TypeRef} +end + +function resolve(ctx::Context, type::TypeRef) + type.kind == :named || return type + haskey(ctx.typedefs, type.name) && return resolve(ctx, ctx.typedefs[type.name]) + haskey(ctx.kinds, type.name) || error("unknown Thrift type $(type.name)") + return type +end + +function jltype(ctx::Context, type::TypeRef) + type = resolve(ctx, type) + type.kind == :base && return BASE_TYPES[type.name] + type.kind == :list && return "Vector{$(jltype(ctx, type.args[1]))}" + type.kind == :set && return "Vector{$(jltype(ctx, type.args[1]))}" + type.kind == :map && return "Vector{Pair{$(jltype(ctx, type.args[1])), $(jltype(ctx, type.args[2]))}}" + ctx.kinds[type.name] == :enum && return "$(type.name).T" + return type.name +end + +function wirecode(ctx::Context, type::TypeRef) + type = resolve(ctx, type) + type.kind == :base && return BASE_CODES[type.name] + type.kind == :list && return "Thrift.LIST" + type.kind == :set && return "Thrift.SET" + type.kind == :map && return "Thrift.MAP" + ctx.kinds[type.name] == :enum && return "Thrift.I32" + return "Thrift.STRUCT" +end + +function isbool(ctx::Context, type::TypeRef) + type = resolve(ctx, type) + return type.kind == :base && type.name == "bool" +end + +function iscontainer(ctx::Context, type::TypeRef) + kind = resolve(ctx, type).kind + return kind == :list || kind == :set || kind == :map +end + +function _checknestedset(ctx::Context, type::TypeRef) + type = resolve(ctx, type) + for arg in type.args + inner = resolve(ctx, arg) + inner.kind == :set && error("nested set types are not supported by the generator") + _checknestedset(ctx, inner) + end + return +end + +function readexpr(ctx::Context, type::TypeRef) + type = resolve(ctx, type) + type.kind == :base && return BASE_READERS[type.name] + ctx.kinds[type.name] == :enum && return "$(type.name).T(Thrift.readi32(r))" + return "Thrift.decode(r, $(type.name))" +end + +function containerreadexpr(ctx::Context, type::TypeRef) + type = resolve(ctx, type) + type.kind == :map && return "Thrift.readmap(r, $(jltype(ctx, type.args[1])), $(jltype(ctx, type.args[2])))" + return "Thrift.readlist(r, $(jltype(ctx, type.args[1])))" +end + +function writestmt(ctx::Context, type::TypeRef, value::String) + type = resolve(ctx, type) + type.kind == :base && return "$(BASE_WRITERS[type.name])(w, $value)" + type.kind == :map && return "Thrift.writemap!(w, $value)" + (type.kind == :list || type.kind == :set) && return "Thrift.writelist!(w, $value)" + ctx.kinds[type.name] == :enum && return "Thrift.writei32!(w, $value.value)" + return "Thrift.encode!(w, $value)" +end + +function defaultexpr(ctx::Context, field::FieldDef) + token = field.default + type = resolve(ctx, field.type) + if type.kind == :base + type.name == "bool" && return token.text + type.name == "string" && return repr(token.text) + type.name == "binary" && return "Vector{UInt8}(codeunits($(repr(token.text))))" + return "$(BASE_TYPES[type.name])($(token.text))" + end + type.kind == :named && ctx.kinds[type.name] == :enum && return token.text + error("unsupported default value for field $(field.name)") +end + +function typetext(type::TypeRef) + type.kind == :list && return "list<$(typetext(type.args[1]))>" + type.kind == :set && return "set<$(typetext(type.args[1]))>" + type.kind == :map && return "map<$(typetext(type.args[1])), $(typetext(type.args[2]))>" + return type.name +end + +function fieldcomment(field::FieldDef) + text = "# $(field.id): " + field.requiredness == :default || (text *= "$(field.requiredness) ") + text *= "$(typetext(field.type)) $(field.name)" + field.default === nothing || (text *= " = $(field.default.text)") + return text +end + +function isrequired(def::StructDef, field::FieldDef) + return def.kind == :struct && field.requiredness == :required +end + +function hasdefault(def::StructDef, field::FieldDef) + return isrequired(def, field) && field.default !== nothing +end + +function mangle(name::String) + occursin('.', name) && error("Thrift identifier $(repr(name)) contains a dot") + name in JULIA_KEYWORDS && return name * "_" + return name +end + +function fieldname(field::FieldDef) + name = mangle(field.name) + name == "unknown_fields" && error("Thrift field name $(repr(field.name)) is reserved") + return name +end + +function localname(field::FieldDef) + return "f_$(fieldname(field))" +end + +function fieldtype(ctx::Context, def::StructDef, field::FieldDef) + _checknestedset(ctx, field.type) + jt = jltype(ctx, field.type) + isrequired(def, field) && return jt + return "Union{Nothing, $jt}" +end + +function emitenum(io::IO, def::EnumDef) + println(io, "module ", mangle(def.name)) + println(io) + println(io, "import ..Thrift") + println(io) + println(io, "struct T <: Thrift.ThriftEnum") + println(io, " value::Int32") + println(io, "end") + for (name, value) in def.values + println(io) + println(io, "const ", mangle(name), " = T(", value, ")") + end + names = join(("(Int32($value), :$(mangle(name)))" for (name, value) in def.values), ", ") + println(io) + println(io, "function Thrift.enumnames(::Core.Type{T})") + println(io, " return (", names, length(def.values) == 1 ? "," : "", ")") + println(io, "end") + println(io) + println(io, "function Thrift.typecode(::Core.Type{T})") + println(io, " return Thrift.I32") + println(io, "end") + println(io) + println(io, "function Thrift.readelement(r::Thrift.Reader, ::Core.Type{T})") + println(io, " return T(Thrift.readi32(r))") + println(io, "end") + println(io) + println(io, "function Thrift.writeelement!(w::Thrift.Writer, x::T)") + println(io, " Thrift.writei32!(w, x.value)") + println(io, " return") + println(io, "end") + println(io) + println(io, "end") + return +end + +function emitstructdef(io::IO, ctx::Context, def::StructDef) + println(io, "# Thrift struct ", def.name) + println(io, "Base.@kwdef struct ", def.name) + for field in def.fields + jt = fieldtype(ctx, def, field) + if hasdefault(def, field) + println(io, " ", fieldname(field), "::", jt, " = ", defaultexpr(ctx, field), " ", fieldcomment(field)) + elseif isrequired(def, field) + println(io, " ", fieldname(field), "::", jt, " ", fieldcomment(field)) + else + println(io, " ", fieldname(field), "::", jt, " = nothing ", fieldcomment(field)) + end + end + println(io, " unknown_fields::Vector{Thrift.RawField} = Thrift.RawField[]") + println(io, "end") + return +end + +function knowncountexpr(def::StructDef) + isempty(def.fields) && return "0" + return join(("($(fieldname(field)) !== nothing)" for field in def.fields), " + ") +end + +function emituniondef(io::IO, ctx::Context, def::StructDef) + names = [fieldname(field) for field in def.fields] + args = join(vcat(names, "unknown_fields"), ", ") + println(io, "# Thrift union ", def.name) + println(io, "struct ", def.name) + for field in def.fields + println(io, " ", fieldname(field), "::", fieldtype(ctx, def, field), " ", fieldcomment(field)) + end + println(io, " unknown_fields::Vector{Thrift.RawField}") + println(io, " function ", def.name, "(", args, ")") + println(io, " Thrift.checkunionargs(:", def.name, ", ", knowncountexpr(def), ", unknown_fields)") + println(io, " return new(", args, ")") + println(io, " end") + println(io, "end") + println(io) + kwargs = join(vcat(["$name=nothing" for name in names], "unknown_fields=Thrift.RawField[]"), ", ") + println(io, "function ", def.name, "(; ", kwargs, ")") + println(io, " return ", def.name, "(", args, ")") + println(io, "end") + return +end + +function emitequality(io::IO, def::StructDef) + names = [fieldname(field) for field in def.fields] + push!(names, "unknown_fields") + println(io) + println(io, "function Base.:(==)(a::", def.name, ", b::", def.name, ")") + println(io, " return ", join(("a.$name == b.$name" for name in names), " && ")) + println(io, "end") + println(io) + println(io, "function Base.isequal(a::", def.name, ", b::", def.name, ")") + println(io, " return ", join(("isequal(a.$name, b.$name)" for name in names), " && ")) + println(io, "end") + println(io) + println(io, "function Base.hash(x::", def.name, ", h::UInt)") + println(io, " h = hash(:", def.name, ", h)") + for name in names + println(io, " h = hash(x.", name, ", h)") + end + println(io, " return h") + println(io, "end") + return +end + +function emitelementhelpers(io::IO, def::StructDef) + println(io) + println(io, "function Thrift.typecode(::Core.Type{", def.name, "})") + println(io, " return Thrift.STRUCT") + println(io, "end") + println(io) + println(io, "function Thrift.readelement(r::Thrift.Reader, ::Core.Type{", def.name, "})") + println(io, " return Thrift.decode(r, ", def.name, ")") + println(io, "end") + println(io) + println(io, "function Thrift.writeelement!(w::Thrift.Writer, x::", def.name, ")") + println(io, " Thrift.encode!(w, x)") + println(io, " return") + println(io, "end") + return +end + +function emitdecodebranch(io::IO, ctx::Context, field::FieldDef, keyword::String) + target = localname(field) + if isbool(ctx, field.type) + println(io, " ", keyword, " id == Int16(", field.id, ") && (ty == Thrift.BOOL_TRUE || ty == Thrift.BOOL_FALSE)") + println(io, " ", target, " = ty == Thrift.BOOL_TRUE") + elseif iscontainer(ctx, field.type) + value = "value_$(fieldname(field))" + println(io, " ", keyword, " id == Int16(", field.id, ") && ty == ", wirecode(ctx, field.type)) + println(io, " ", value, " = ", containerreadexpr(ctx, field.type)) + println(io, " if ", value, " === nothing") + println(io, " unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty))") + println(io, " else") + println(io, " ", target, " = ", value) + println(io, " end") + else + println(io, " ", keyword, " id == Int16(", field.id, ") && ty == ", wirecode(ctx, field.type)) + println(io, " ", target, " = ", readexpr(ctx, field.type)) + end + return +end + +function emitdecode(io::IO, ctx::Context, def::StructDef) + println(io) + println(io, "function Thrift.decode(r::Thrift.Reader, ::Core.Type{", def.name, "})") + println(io, " Thrift.enter!(r)") + for field in def.fields + init = hasdefault(def, field) ? defaultexpr(ctx, field) : "nothing" + println(io, " ", localname(field), " = ", init) + end + println(io, " unknown = nothing") + println(io, " lastid = Int16(0)") + println(io, " while true") + println(io, " id, ty = Thrift.readfieldheader(r, lastid)") + println(io, " ty == Thrift.STOP && break") + println(io, " lastid = id") + for (index, field) in enumerate(def.fields) + emitdecodebranch(io, ctx, field, index == 1 ? "if" : "elseif") + end + isempty(def.fields) || println(io, " else") + indent = isempty(def.fields) ? " " : " " + println(io, indent, "unknown = Thrift.pushunknown!(unknown, Thrift.readrawfield(r, id, ty))") + isempty(def.fields) || println(io, " end") + println(io, " end") + println(io, " Thrift.leave!(r)") + println(io, " unknown_fields = Thrift.finishunknown(r, unknown)") + for field in def.fields + isrequired(def, field) && !hasdefault(def, field) || continue + println(io, " ", localname(field), " === nothing && Thrift.missingfield(:", def.name, ", :", field.name, ")") + end + if def.kind == :union + known = isempty(def.fields) ? "0" : join(("($(localname(field)) !== nothing)" for field in def.fields), " + ") + println(io, " Thrift.checkunion(:", def.name, ", ", known, ", unknown_fields)") + end + args = join((localname(field) for field in def.fields), ", ") + isempty(def.fields) || (args *= ", ") + println(io, " return ", def.name, "(", args, "unknown_fields)") + println(io, "end") + return +end + +function emitencodefield(io::IO, ctx::Context, def::StructDef, field::FieldDef) + value = "x.$(fieldname(field))" + indent = " " + if !isrequired(def, field) + value = "value_$(fieldname(field))" + println(io, " ", value, " = x.", fieldname(field)) + println(io, " if ", value, " !== nothing") + indent = " " + end + code = isbool(ctx, field.type) ? "$value ? Thrift.BOOL_TRUE : Thrift.BOOL_FALSE" : wirecode(ctx, field.type) + println(io, indent, "lastid = Thrift.writefieldheader!(w, lastid, Int16(", field.id, "), ", code, ")") + isbool(ctx, field.type) || println(io, indent, writestmt(ctx, field.type, value)) + println(io, indent, "(lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid)") + isrequired(def, field) || println(io, " end") + return +end + +function emitencode(io::IO, ctx::Context, def::StructDef) + println(io) + println(io, "function Thrift.encode!(w::Thrift.Writer, x::", def.name, ")") + println(io, " unknown = x.unknown_fields") + if def.kind == :union + known = isempty(def.fields) ? "0" : + join(("(x.$(fieldname(field)) !== nothing)" for field in def.fields), + " + ") + println(io, " Thrift.checkunionargs(:", def.name, ", ", + known, ", unknown)") + end + println(io, " lastid = Int16(0)") + println(io, " index = 1") + println(io, " (lastid, index) = Thrift.writeunknownafter!(w, unknown, index, lastid)") + for field in sort(def.fields; by=field -> field.id) + emitencodefield(io, ctx, def, field) + end + println(io, " Thrift.writeunknownrest!(w, unknown, index, lastid)") + println(io, " Thrift.writestop!(w)") + println(io, " return") + println(io, "end") + return +end + +function emitstruct(io::IO, ctx::Context, def::StructDef) + mangle(def.name) == def.name || error("Thrift struct name $(repr(def.name)) is a Julia keyword") + ids = [field.id for field in def.fields] + allunique(ids) || error("duplicate field ids in $(def.name)") + allunique(fieldname(field) for field in def.fields) || error("duplicate field names in $(def.name)") + def.kind == :union ? emituniondef(io, ctx, def) : emitstructdef(io, ctx, def) + emitequality(io, def) + emitelementhelpers(io, def) + emitdecode(io, ctx, def) + emitencode(io, ctx, def) + return +end + +function fnv1a64(bytes::AbstractVector{UInt8}) + h = 0xcbf29ce484222325 + for byte in bytes + h = (h ⊻ UInt64(byte)) * 0x00000100000001b3 + end + return h +end + +function buildcontext(definitions::Vector{Any}) + ctx = Context(Dict{String,Symbol}(), Dict{String,TypeRef}()) + for def in definitions + haskey(ctx.kinds, def.name) && error("duplicate definition $(def.name)") + if def isa EnumDef + ctx.kinds[def.name] = :enum + elseif def isa StructDef + ctx.kinds[def.name] = :struct + else + ctx.kinds[def.name] = :typedef + ctx.typedefs[def.name] = def.type + end + end + return ctx +end + +""" + generate(idl::String; version, commit) -> String + +Generate the Julia source of the `Metadata` module from Thrift IDL text. +""" +function generate(idl::String; version::String=FORMAT_VERSION, commit::String=FORMAT_COMMIT) + definitions = parsedocument(tokenize(idl)) + ctx = buildcontext(definitions) + io = IOBuffer() + println(io, "# Generated by thrift/generate.jl from thrift/parquet.thrift. Do not edit by hand.") + println(io, "# Source: apache/parquet-format ", version, " (", commit, ")") + println(io, "# IDL: ", sizeof(idl), " bytes, FNV-1a 64 0x", string(fnv1a64(codeunits(idl)); base=16, pad=16)) + println(io, "module Metadata") + println(io) + println(io, "import ..Thrift") + for def in definitions + def isa TypedefDef && continue + println(io) + def isa EnumDef ? emitenum(io, def) : emitstruct(io, ctx, def) + end + println(io) + println(io, "end") + return String(take!(io)) +end + +function main(args::Vector{String}) + output = generate(read(IDL_PATH, String)) + if "--check" in args + existing = isfile(OUTPUT_PATH) ? read(OUTPUT_PATH, String) : "" + existing == output && (println("src/metadata/parquet.jl is up to date"); return 0) + println(stderr, "src/metadata/parquet.jl is stale; run julia thrift/generate.jl") + return 1 + end + mkpath(dirname(OUTPUT_PATH)) + write(OUTPUT_PATH, output) + println("wrote ", OUTPUT_PATH) + return 0 +end + +end + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(ThriftGenerator.main(ARGS)) +end diff --git a/thrift/genproto.sh b/thrift/genproto.sh deleted file mode 100755 index 902c101..0000000 --- a/thrift/genproto.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -OUTDIR=${DIR}/../src/ -thrift --gen jl --out ${OUTDIR} PAR2.thrift diff --git a/thrift/PAR2.thrift b/thrift/parquet.thrift similarity index 54% rename from thrift/PAR2.thrift rename to thrift/parquet.thrift index e480fd8..fe259d6 100644 --- a/thrift/PAR2.thrift +++ b/thrift/parquet.thrift @@ -17,8 +17,6 @@ * under the License. */ -cpp_include "parquet/util/windows_compatibility.h" - /** * File format description for the parquet file format */ @@ -35,7 +33,7 @@ enum Type { BOOLEAN = 0; INT32 = 1; INT64 = 2; - INT96 = 3; // deprecated, only used by legacy implementations. + INT96 = 3; // deprecated, new Parquet writers should not write data in INT96 FLOAT = 4; DOUBLE = 5; BYTE_ARRAY = 6; @@ -43,9 +41,10 @@ enum Type { } /** - * Common types used by frameworks(e.g. hive, pig) using parquet. This helps map - * between types in those frameworks to the base types in parquet. This is only - * metadata and not needed to read or write the data. + * DEPRECATED: Common types used by frameworks (e.g. Hive, Pig) using parquet. + * ConvertedType is superseded by LogicalType. This enum should not be extended. + * + * See LogicalTypes.md for conversion between ConvertedType and LogicalType. */ enum ConvertedType { /** a BYTE_ARRAY actually contains UTF8 encoded chars */ @@ -61,14 +60,14 @@ enum ConvertedType { * values */ LIST = 3; - /** an enum is converted into a binary field */ + /** an enum is converted into a BYTE_ARRAY field */ ENUM = 4; /** * A decimal value. * - * This may be used to annotate binary or fixed primitive types. The - * underlying byte array stores the unscaled value encoded as two's + * This may be used to annotate BYTE_ARRAY or FIXED_LEN_BYTE_ARRAY primitive + * types. The underlying byte array stores the unscaled value encoded as two's * complement using big-endian byte order (the most significant byte is the * zeroth element). The value of the decimal is the value * 10^{-scale}. * @@ -159,7 +158,7 @@ enum ConvertedType { /** * An embedded BSON document * - * A BSON document embedded within a single BINARY column. + * A BSON document embedded within a single BYTE_ARRAY column. */ BSON = 20; @@ -182,16 +181,85 @@ enum ConvertedType { * Representation of Schemas */ enum FieldRepetitionType { - /** This field is required (can not be null) and each record has exactly 1 value. */ + /** This field is required (can not be null) and each row has exactly 1 value. */ REQUIRED = 0; - /** The field is optional (can be null) and each record has 0 or 1 values. */ + /** The field is optional (can be null) and each row has 0 or 1 values. */ OPTIONAL = 1; /** The field is repeated and can contain 0 or more values */ REPEATED = 2; } +/** + * A structure for capturing metadata for estimating the unencoded, + * uncompressed size of data written. This is useful for readers to estimate + * how much memory is needed to reconstruct data in their memory model and for + * fine grained filter pushdown on nested structures (the histograms contained + * in this structure can help determine the number of nulls at a particular + * nesting level and maximum length of lists). + */ +struct SizeStatistics { + /** + * The number of physical bytes stored for BYTE_ARRAY data values assuming + * no encoding. This is exclusive of the bytes needed to store the length of + * each byte array. In other words, this field is equivalent to the `(size + * of PLAIN-ENCODING the byte array values) - (4 bytes * number of values + * written)`. To determine unencoded sizes of other types readers can use + * schema information multiplied by the number of non-null and null values. + * The number of null/non-null values can be inferred from the histograms + * below. + * + * For example, if a column chunk is dictionary-encoded with dictionary + * ["a", "bc", "cde"], and a data page contains the indices [0, 0, 1, 2], + * then this value for that data page should be 7 (1 + 1 + 2 + 3). + * + * This field should only be set for types that use BYTE_ARRAY as their + * physical type. + */ + 1: optional i64 unencoded_byte_array_data_bytes; + /** + * When present, there is expected to be one element corresponding to each + * repetition (i.e. size=max repetition_level+1) where each element + * represents the number of times the repetition level was observed in the + * data. + * + * This field may be omitted if max_repetition_level is 0 without loss + * of information. + **/ + 2: optional list repetition_level_histogram; + /** + * Same as repetition_level_histogram except for definition levels. + * + * This field may be omitted if max_definition_level is 0 or 1 without + * loss of information. + **/ + 3: optional list definition_level_histogram; +} + +/** + * Bounding box for GEOMETRY or GEOGRAPHY type in the representation of min/max + * value pair of coordinates from each axis. + */ +struct BoundingBox { + 1: required double xmin; + 2: required double xmax; + 3: required double ymin; + 4: required double ymax; + 5: optional double zmin; + 6: optional double zmax; + 7: optional double mmin; + 8: optional double mmax; +} + +/** Statistics specific to Geometry and Geography logical types */ +struct GeospatialStatistics { + /** A bounding box of geospatial instances */ + 1: optional BoundingBox bbox; + /** Geospatial type codes of all instances, or an empty list if not known */ + 2: optional list geospatial_types; +} + /** * Statistics per row group and per page * All fields are optional. @@ -212,27 +280,52 @@ struct Statistics { */ 1: optional binary max; 2: optional binary min; - /** count of null value in the column */ + /** + * Count of null values in the column. + * + * Writers SHOULD always write this field even if it is zero (i.e. no null value) + * or the column is not nullable. + * Readers MUST distinguish between null_count not being present and null_count == 0. + * If null_count is not present, readers MUST NOT assume null_count == 0. + */ 3: optional i64 null_count; /** count of distinct values occurring */ 4: optional i64 distinct_count; /** - * Min and max values for the column, determined by its ColumnOrder. + * Lower and upper bound values for the column, determined by its ColumnOrder. + * + * These may be the actual minimum and maximum values found on a page or column + * chunk, but can also be (more compact) values that do not exist on a page or + * column chunk. For example, instead of storing "Blart Versenwald III", a writer + * may set min_value="B", max_value="C". Such more compact values must still be + * valid values within the column's logical type. * * Values are encoded using PLAIN encoding, except that variable-length byte * arrays do not include a length prefix. */ 5: optional binary max_value; 6: optional binary min_value; + /** If true, max_value is the actual maximum value for a column */ + 7: optional bool is_max_value_exact; + /** If true, min_value is the actual minimum value for a column */ + 8: optional bool is_min_value_exact; + /** + * Count of NaN values in the column; only present if physical type is FLOAT + * or DOUBLE, or logical type is FLOAT16. + * If this field is not present, readers MUST assume NaNs may be present + * (i.e. MUST assume nan_count > 0 and MAY NOT assume nan_count == 0). + */ + 9: optional i64 nan_count; } /** Empty structs to use as logical type annotations */ -struct StringType {} // allowed for BINARY, must be encoded with UTF-8 -struct UUIDType {} // allowed for FIXED[16], must encoded raw UUID bytes +struct StringType {} // allowed for BYTE_ARRAY, must be encoded with UTF-8 +struct UUIDType {} // allowed for FIXED[16], must be encoded as raw UUID bytes struct MapType {} // see LogicalTypes.md struct ListType {} // see LogicalTypes.md -struct EnumType {} // allowed for BINARY, must be encoded with UTF-8 +struct EnumType {} // allowed for BYTE_ARRAY, must be encoded with UTF-8 struct DateType {} // allowed for INT32 +struct Float16Type {} // allowed for FIXED[2], must be encoded as raw FLOAT16 bytes (see LogicalTypes.md) /** * Logical type to annotate a column that is always null. @@ -246,10 +339,13 @@ struct NullType {} // allowed for any physical type, only null values stored /** * Decimal logical type annotation * + * Scale must be zero or a positive integer less than or equal to the precision. + * Precision must be a non-zero positive integer. + * * To maintain forward-compatibility in v1, implementations using this logical * type must also set scale and precision on the annotated SchemaElement. * - * Allowed for physical types: INT32, INT64, FIXED, and BINARY + * Allowed for physical types: INT32, INT64, FIXED_LEN_BYTE_ARRAY, and BYTE_ARRAY. */ struct DecimalType { 1: required i32 scale @@ -301,7 +397,7 @@ struct IntType { /** * Embedded JSON logical type annotation * - * Allowed for physical types: BINARY + * Allowed for physical types: BYTE_ARRAY */ struct JsonType { } @@ -309,24 +405,82 @@ struct JsonType { /** * Embedded BSON logical type annotation * - * Allowed for physical types: BINARY + * Allowed for physical types: BYTE_ARRAY */ struct BsonType { } +/** + * Embedded Variant logical type annotation + */ +struct VariantType { + // The version of the variant specification that the variant was + // written with. + 1: optional i8 specification_version +} + +/** Edge interpolation algorithm for Geography logical type */ +enum EdgeInterpolationAlgorithm { + SPHERICAL = 0; + VINCENTY = 1; + THOMAS = 2; + ANDOYER = 3; + KARNEY = 4; +} + +/** + * Embedded Geometry logical type annotation + * + * Geospatial features in the Well-Known Binary (WKB) format and `edges` interpolation + * is always linear/planar. + * + * A custom CRS can be set by the crs field. If unset, it defaults to "OGC:CRS84", + * which means that the geometries must be stored in longitude, latitude based on + * the WGS84 datum. + * + * Allowed for physical type: BYTE_ARRAY. + * + * See Geospatial.md for details. + */ +struct GeometryType { + 1: optional string crs; +} + +/** + * Embedded Geography logical type annotation + * + * Geospatial features in the WKB format with an explicit (non-linear/non-planar) + * `edges` interpolation algorithm. + * + * A custom geographic CRS can be set by the crs field, where longitudes are + * bound by [-180, 180] and latitudes are bound by [-90, 90]. If unset, the CRS + * defaults to "OGC:CRS84". + * + * An optional algorithm can be set to correctly interpret `edges` interpolation + * of the geometries. If unset, the algorithm defaults to SPHERICAL. + * + * Allowed for physical type: BYTE_ARRAY. + * + * See Geospatial.md for details. + */ +struct GeographyType { + 1: optional string crs; + 2: optional EdgeInterpolationAlgorithm algorithm; +} + /** * LogicalType annotations to replace ConvertedType. * * To maintain compatibility, implementations using LogicalType for a - * SchemaElement must also set the corresponding ConvertedType from the - * following table. + * SchemaElement must also set the corresponding ConvertedType (if any) + * from the following table. */ union LogicalType { 1: StringType STRING // use ConvertedType UTF8 2: MapType MAP // use ConvertedType MAP 3: ListType LIST // use ConvertedType LIST 4: EnumType ENUM // use ConvertedType ENUM - 5: DecimalType DECIMAL // use ConvertedType DECIMAL + 5: DecimalType DECIMAL // use ConvertedType DECIMAL + SchemaElement.{scale, precision} 6: DateType DATE // use ConvertedType DATE // use ConvertedType TIME_MICROS for TIME(isAdjustedToUTC = *, unit = MICROS) @@ -342,11 +496,15 @@ union LogicalType { 11: NullType UNKNOWN // no compatible ConvertedType 12: JsonType JSON // use ConvertedType JSON 13: BsonType BSON // use ConvertedType BSON - 14: UUIDType UUID + 14: UUIDType UUID // no compatible ConvertedType + 15: Float16Type FLOAT16 // no compatible ConvertedType + 16: VariantType VARIANT // no compatible ConvertedType + 17: GeometryType GEOMETRY // no compatible ConvertedType + 18: GeographyType GEOGRAPHY // no compatible ConvertedType } /** - * Represents a element inside a schema definition. + * Represents an element inside a schema definition. * - if it is a group (inner node) then type is undefined and num_children is defined * - if it is a primitive type (leaf) then type is defined and num_children is undefined * the nodes are listed in depth first traversal order. @@ -355,7 +513,7 @@ struct SchemaElement { /** Data type for this field. Not set if the current element is a non-leaf node */ 1: optional Type type; - /** If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales. + /** If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values. * Otherwise, if specified, this is the maximum bit length to store any of the values. * (e.g. a low cardinality INT col could have this set to 3). Note that this is * in the schema, and therefore fixed for the entire file. @@ -376,13 +534,19 @@ struct SchemaElement { */ 5: optional i32 num_children; - /** When the schema is the result of a conversion from another model + /** + * DEPRECATED: When the schema is the result of a conversion from another model. * Used to record the original type to help with cross conversion. + * + * This is superseded by logicalType. */ 6: optional ConvertedType converted_type; - /** Used when this column contains decimal data. + /** + * DEPRECATED: Used when this column contains decimal data. * See the DECIMAL converted type for more details. + * + * This is superseded by using the DecimalType annotation in logicalType. */ 7: optional i32 scale 8: optional i32 precision @@ -419,15 +583,15 @@ enum Encoding { PLAIN = 0; /** Group VarInt encoding for INT32/INT64. - * This encoding is deprecated. It was never used + * This encoding is deprecated. It was never used. */ // GROUP_VAR_INT = 1; /** - * Deprecated: Dictionary encoding. The values in the dictionary are encoded in the + * DEPRECATED: Dictionary encoding. The values in the dictionary are encoded in the * plain type. - * in a data page use RLE_DICTIONARY instead. - * in a Dictionary page use PLAIN instead + * For a data page use RLE_DICTIONARY instead. + * For a Dictionary page use PLAIN instead. */ PLAIN_DICTIONARY = 2; @@ -436,8 +600,9 @@ enum Encoding { */ RLE = 3; - /** Bit packed encoding. This can only be used if the data has a known max + /** DEPRECATED: Bit packed encoding. This can only be used if the data has a known max * width. Usable for definition/repetition levels encoding. + * Superseded by RLE (which is a hybrid of RLE and bit packing); see Encodings.md. */ BIT_PACKED = 4; @@ -460,12 +625,15 @@ enum Encoding { */ RLE_DICTIONARY = 8; - /** Encoding for floating-point data. + /** Encoding for fixed-width data (FLOAT, DOUBLE, INT32, INT64, FIXED_LEN_BYTE_ARRAY). K byte-streams are created where K is the size in bytes of the data type. - The individual bytes of an FP value are scattered to the corresponding stream and + The individual bytes of a value are scattered to the corresponding stream and the streams are concatenated. This itself does not reduce the size of the data but can lead to better compression afterwards. + + Added in 2.8 for FLOAT and DOUBLE. + Support for INT32, INT64 and FIXED_LEN_BYTE_ARRAY added in 2.11. */ BYTE_STREAM_SPLIT = 9; } @@ -473,19 +641,21 @@ enum Encoding { /** * Supported compression algorithms. * - * Codecs added in 2.4 can be read by readers based on 2.4 and later. + * Codecs added in format version X.Y can be read by readers based on X.Y and later. * Codec support may vary between readers based on the format version and - * libraries available at runtime. Gzip, Snappy, and LZ4 codecs are - * widely available, while Zstd and Brotli require additional libraries. + * libraries available at runtime. + * + * See Compression.md for a detailed specification of these algorithms. */ enum CompressionCodec { UNCOMPRESSED = 0; SNAPPY = 1; GZIP = 2; LZO = 3; - BROTLI = 4; // Added in 2.4 - LZ4 = 5; // Added in 2.4 - ZSTD = 6; // Added in 2.4 + BROTLI = 4; // Added in 2.4 + LZ4 = 5; // DEPRECATED (Added in 2.4) + ZSTD = 6; // Added in 2.4 + LZ4_RAW = 7; // Added in 2.9 } enum PageType { @@ -507,7 +677,13 @@ enum BoundaryOrder { /** Data page header */ struct DataPageHeader { - /** Number of values, including NULLs, in this data page. **/ + /** + * Number of values, including NULLs, in this data page. + * + * If an OffsetIndex is present, a page must begin at a row + * boundary (repetition_level = 0). Otherwise, pages may begin + * within a row (repetition_level > 0). + **/ 1: required i32 num_values /** Encoding used for this data page **/ @@ -519,7 +695,7 @@ struct DataPageHeader { /** Encoding used for repetition levels **/ 4: required Encoding repetition_level_encoding; - /** Optional statistics for the data in this page**/ + /** Optional statistics for the data in this page **/ 5: optional Statistics statistics; } @@ -527,6 +703,11 @@ struct IndexPageHeader { // TODO } +/** + * The dictionary page must be placed at the first position of the column chunk + * if it is partly or completely dictionary encoded. At most one dictionary page + * can be placed in a column chunk. + **/ struct DictionaryPageHeader { /** Number of values in the dictionary **/ 1: required i32 num_values; @@ -539,9 +720,14 @@ struct DictionaryPageHeader { } /** - * New page format allowing reading levels without decompressing the data + * Alternate page format allowing reading levels without decompressing the data * Repetition and definition levels are uncompressed * The remaining section containing the data is compressed if is_compressed is true + * + * Implementation note - this header is not necessarily a strict improvement over + * `DataPageHeader` (in particular the original header might provide better compression + * in some scenarios). Page indexes require pages to start and end at row boundaries, + * regardless of which page header is used. **/ struct DataPageHeaderV2 { /** Number of values, including NULLs, in this data page. **/ @@ -549,26 +735,30 @@ struct DataPageHeaderV2 { /** Number of NULL values, in this data page. Number of non-null = num_values - num_nulls which is also the number of values in the data section **/ 2: required i32 num_nulls - /** Number of rows in this data page. which means pages change on record boundaries (r = 0) **/ + /** + * Number of rows in this data page. Every page must begin at a + * row boundary (repetition_level = 0): rows must **not** be + * split across page boundaries when using V2 data pages. + **/ 3: required i32 num_rows /** Encoding used for data in this page **/ 4: required Encoding encoding // repetition levels and definition levels are always using RLE (without size in it) - /** length of the definition levels */ + /** Length of the definition levels */ 5: required i32 definition_levels_byte_length; - /** length of the repetition levels */ + /** Length of the repetition levels */ 6: required i32 repetition_levels_byte_length; - /** whether the values are compressed. + /** Whether the values are compressed. Which means the section of the page between - definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included) + definition_levels_byte_length + repetition_levels_byte_length and compressed_page_size (included) is compressed with the compression_codec. If missing it is considered compressed */ - 7: optional bool is_compressed = 1; + 7: optional bool is_compressed = true; - /** optional statistics for this column chunk */ + /** Optional statistics for the data in this page **/ 8: optional Statistics statistics; } @@ -627,30 +817,23 @@ struct PageHeader { /** Compressed (and potentially encrypted) page size in bytes, not including this header **/ 3: required i32 compressed_page_size - /** The 32bit CRC for the page, to be be calculated as follows: - * - Using the standard CRC32 algorithm - * - On the data only, i.e. this header should not be included. 'Data' - * hereby refers to the concatenation of the repetition levels, the - * definition levels and the column value, in this exact order. - * - On the encoded versions of the repetition levels, definition levels and - * column values - * - On the compressed versions of the repetition levels, definition levels - * and column values where possible; - * - For v1 data pages, the repetition levels, definition levels and column - * values are always compressed together. If a compression scheme is - * specified, the CRC shall be calculated on the compressed version of - * this concatenation. If no compression scheme is specified, the CRC - * shall be calculated on the uncompressed version of this concatenation. - * - For v2 data pages, the repetition levels and definition levels are - * handled separately from the data and are never compressed (only - * encoded). If a compression scheme is specified, the CRC shall be - * calculated on the concatenation of the uncompressed repetition levels, - * uncompressed definition levels and the compressed column values. - * If no compression scheme is specified, the CRC shall be calculated on - * the uncompressed concatenation. + /** The 32-bit CRC checksum for the page, to be calculated as follows: + * + * - The standard CRC32 algorithm is used (with polynomial 0x04C11DB7, + * the same as in e.g. GZIP). + * - All page types can have a CRC (v1 and v2 data pages, dictionary pages, + * etc.). + * - The CRC is computed on the serialization binary representation of the page + * (as written to disk), excluding the page header. For example, for v1 + * data pages, the CRC is computed on the concatenation of repetition levels, + * definition levels and column values (optionally compressed, optionally + * encrypted). + * - The CRC computation therefore takes place after any compression + * and encryption steps, if any. + * * If enabled, this allows for disabling checksumming in HDFS if only a few * pages need to be read. - **/ + */ 4: optional i32 crc // Headers for page specific data. One only will be set. @@ -669,10 +852,10 @@ struct PageHeader { } /** - * Wrapper struct to specify sort order + * Sort order within a RowGroup of a leaf column */ struct SortingColumn { - /** The column index (in this row group) **/ + /** The ordinal position of the column (in this row group) **/ 1: required i32 column_idx /** If true, indicates this column is sorted in descending order. **/ @@ -748,6 +931,25 @@ struct ColumnMetaData { /** Byte offset from beginning of file to Bloom filter data. **/ 14: optional i64 bloom_filter_offset; + + /** Size of Bloom filter data including the serialized header, in bytes. + * Added in 2.10 so readers may not read this field from old files and + * it can be obtained after the BloomFilterHeader has been deserialized. + * Writers should write this field so readers can read the bloom filter + * in a single I/O. + */ + 15: optional i32 bloom_filter_length; + + /** + * Optional statistics to help estimate total memory when converted to in-memory + * representations. The histograms contained in these statistics can + * also be useful in some cases for more fine-grained nullability/list length + * filter pushdown. + */ + 16: optional SizeStatistics size_statistics; + + /** Optional statistics specific for Geometry and Geography logical types */ + 17: optional GeospatialStatistics geospatial_statistics; } struct EncryptionWithFooterKey { @@ -769,15 +971,39 @@ union ColumnCryptoMetaData { struct ColumnChunk { /** File where column data is stored. If not set, assumed to be same file as * metadata. This path is relative to the current file. + * + * As of December 2025, the only known use-case for this field is writing summary + * parquet files (i.e. "_metadata" files). These files consolidate footers from + * multiple parquet files to allow for efficient reading of footers to avoid file + * listing costs and prune out files that do not need to be read based on statistics. + * + * These files do not appear to have ever been formally specified in the specification. + * and are potentially problematic from a correctness perspective [1]. + * + * [1] https://lists.apache.org/thread/ootf2kmyg3p01b1bvplpvp4ftd1bt72d + * + * There is no other known usage of this field. Specifically, there are no known + * reference implementations that will read externally stored column data if this field is populated + * within a standard parquet file. Making use of the field for this purpose is + * not considered part of the Parquet specification. **/ 1: optional string file_path - /** Byte offset in file_path to the ColumnMetaData **/ - 2: required i64 file_offset + /** DEPRECATED: Byte offset in file_path to the ColumnMetaData + * + * Past use of this field has been inconsistent, with some implementations + * using it to point to the ColumnMetaData and some using it to point to + * the first page in the column chunk. In many cases, the ColumnMetaData at this + * location is wrong. This field is now deprecated and should not be used. + * Writers should set this field to 0 if no ColumnMetaData has been written outside + * the footer. + */ + 2: required i64 file_offset = 0 - /** Column metadata for this chunk. This is the same content as what is at - * file_path/file_offset. Having it here has it replicated in the file - * metadata. + /** Column metadata for this chunk. Some writers may also replicate this at the + * location pointed to by file_path/file_offset. + * Note: while marked as optional, this field is in fact required by most major + * Parquet implementations. As such, writers MUST populate this field. **/ 3: optional ColumnMetaData meta_data @@ -832,6 +1058,9 @@ struct RowGroup { /** Empty struct to signal the order defined by the physical or logical type */ struct TypeDefinedOrder {} +/** Empty struct to signal IEEE 754 total order for floating point types */ +struct IEEE754TotalOrder {} + /** * Union to specify the order used for the min_value and max_value fields for a * column. This union takes the role of an enhanced enum that allows rich @@ -840,6 +1069,7 @@ struct TypeDefinedOrder {} * Possible values are: * * TypeDefinedOrder - the column uses the order defined by its logical or * physical type (if there is no logical type). + * * IEEE754TotalOrder - the floating point column uses IEEE 754 total order. * * If the reader does not support the value of this union, min and max stats * for this column should be ignored. @@ -859,37 +1089,112 @@ union ColumnOrder { * UINT64 - unsigned comparison * DECIMAL - signed comparison of the represented value * DATE - signed comparison + * FLOAT16 - signed comparison of the represented value (*) * TIME_MILLIS - signed comparison * TIME_MICROS - signed comparison * TIMESTAMP_MILLIS - signed comparison * TIMESTAMP_MICROS - signed comparison - * INTERVAL - unsigned comparison + * INTERVAL - undefined * JSON - unsigned byte-wise comparison * BSON - unsigned byte-wise comparison * ENUM - unsigned byte-wise comparison * LIST - undefined * MAP - undefined + * VARIANT - undefined + * GEOMETRY - undefined + * GEOGRAPHY - undefined * * In the absence of logical types, the sort order is determined by the physical type: * BOOLEAN - false, true * INT32 - signed comparison * INT64 - signed comparison - * INT96 (only used for legacy timestamps) - undefined + * INT96 (only used for legacy timestamps) - undefined(+) * FLOAT - signed comparison of the represented value (*) * DOUBLE - signed comparison of the represented value (*) * BYTE_ARRAY - unsigned byte-wise comparison * FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison * - * (*) Because the sorting order is not specified properly for floating - * point values (relations vs. total ordering) the following + * (+) While the INT96 type has been deprecated, at the time of writing it is + * still used in many legacy systems. If a Parquet implementation chooses + * to write statistics for INT96 columns, it is recommended to order them + * according to the legacy rules: + * - compare the last 4 bytes (days) as a little-endian 32-bit signed integer + * - if equal last 4 bytes, compare the first 8 bytes as a little-endian + * 64-bit signed integer (nanos) + * See https://github.com/apache/parquet-format/issues/502 for more details + * + * (*) Because TYPE_ORDER is ambiguous for floating point types due to + * underspecified handling of NaN and -0/+0, it is recommended that writers + * use IEEE_754_TOTAL_ORDER for these types. + * + * If TYPE_ORDER is used for floating point types, then the following * compatibility rules should be applied when reading statistics: * - If the min is a NaN, it should be ignored. * - If the max is a NaN, it should be ignored. + * - If the nan_count field is set, a reader can compute + * nan_count + null_count == num_values to deduce whether all non-null + * values are NaN. * - If the min is +0, the row group may contain -0 values as well. * - If the max is -0, the row group may contain +0 values as well. * - When looking for NaN values, min and max should be ignored. + * If the nan_count field is set, it can be used to check whether + * NaNs are present. + * + * When writing page or column chunk statistics for columns with + * TYPE_ORDER order, the following rules must be followed: + * - The nan_count field must be set for floating point types, even if + * it is zero. + * - If the nan_count field is set, min and max statistics fields, when + * present, must not contain NaN values and must be computed from + * non-NaN values only. This signals to readers that the min and max + * statistics are reliable for non-NaN values. + * - If all non-null values are NaN, min and max statistics must not be + * written. + * - If the computed max value is zero (whether negative or positive), + * `+0.0` should be written into the max statistics field. + * - If the computed min value is zero (whether negative or positive), + * `-0.0` should be written into the min statistics field. + * + * When writing column indexes for columns with TYPE_ORDER order, the + * following rules must be followed: + * - NaNs must not be written to min_values or max_values. + * - If all non-null values of a page are NaN, a column index must not + * be written for this column chunk because min_values and max_values + * are required. + * - If the computed max value is zero (whether negative or positive), + * `+0.0` should be written into the corresponding max_values entry. + * - If the computed min value is zero (whether negative or positive), + * `-0.0` should be written into the corresponding min_values entry. */ 1: TypeDefinedOrder TYPE_ORDER; + + /* + * The floating point type is ordered according to the totalOrder predicate, + * as defined in section 5.10 of IEEE-754 (2008 revision). Only columns of + * physical type FLOAT or DOUBLE, or logical type FLOAT16 may use this ordering. + * + * Intuitively, this orders floats mathematically, but defines -0 to be less + * than +0, -NaN to be less than anything else, and +NaN to be greater than + * anything else. It also defines an order between different bit representations + * of the same value. + * + * When writing statistics for columns with IEEE_754_TOTAL_ORDER order, then + * following rules must be followed: + * - Writing the nan_count field is mandatory when using this ordering. + * - Min and max statistics must contain the smallest and largest non-NaN + * values respectively, or if all non-null values are NaN, the smallest and + * largest NaN values as defined by IEEE 754 total order. + * + * When reading statistics for columns with this order, the following rules + * should be followed: + * - Readers should consult the nan_count field to determine whether NaNs + * are present. + * - A reader can compute nan_count + null_count == num_values to deduce + * whether all non-null values are NaN. In the page index, which does not + * have a num_values field, the presence of a NaN value in min_values + * or max_values indicates that all non-null values are NaN. + */ + 2: IEEE754TotalOrder IEEE_754_TOTAL_ORDER; } struct PageLocation { @@ -897,29 +1202,50 @@ struct PageLocation { 1: required i64 offset /** - * Size of the page, including header. Sum of compressed_page_size and header - * length + * Size of the page, including header. Equal to the sum of the page's + * PageHeader.compressed_page_size and the size of the serialized PageHeader. */ 2: required i32 compressed_page_size /** - * Index within the RowGroup of the first row of the page; this means pages - * change on record boundaries (r = 0). + * Index within the RowGroup of the first row of the page. When an + * OffsetIndex is present, pages must begin on row boundaries + * (repetition_level = 0). */ 3: required i64 first_row_index } +/** + * Optional offsets for each data page in a ColumnChunk. + * + * Forms part of the page index, along with ColumnIndex. + * + * OffsetIndex may be present even if ColumnIndex is not. + */ struct OffsetIndex { /** * PageLocations, ordered by increasing PageLocation.offset. It is required * that page_locations[i].first_row_index < page_locations[i+1].first_row_index. */ 1: required list page_locations + /** + * Unencoded/uncompressed size for BYTE_ARRAY types. + * + * See documentation for unencoded_byte_array_data_bytes in SizeStatistics for + * more details on this field. + */ + 2: optional list unencoded_byte_array_data_bytes } /** - * Description for ColumnIndex. - * Each [i] refers to the page at OffsetIndex.page_locations[i] + * Optional statistics for each data page in a ColumnChunk. + * + * Forms part the page index, along with OffsetIndex. + * + * If this structure is present, OffsetIndex must also be present. + * + * For each field in this structure, [i] refers to the page at + * OffsetIndex.page_locations[i] */ struct ColumnIndex { /** @@ -932,27 +1258,76 @@ struct ColumnIndex { 1: required list null_pages /** - * Two lists containing lower and upper bounds for the values of each page. - * These may be the actual minimum and maximum values found on a page, but - * can also be (more compact) values that do not exist on a page. For - * example, instead of storing ""Blart Versenwald III", a writer may set - * min_values[i]="B", max_values[i]="C". Such more compact values must still - * be valid values within the column's logical type. Readers must make sure - * that list entries are populated before using them by inspecting null_pages. + * Two lists containing lower and upper bounds for the values of each page + * determined by the ColumnOrder of the column. These may be the actual + * minimum and maximum values found on a page, but can also be (more compact) + * values that do not exist on a page. For example, instead of storing "Blart + * Versenwald III", a writer may set min_values[i]="B", max_values[i]="C". + * Such more compact values must still be valid values within the column's + * logical type. Readers must make sure that list entries are populated before + * using them by inspecting null_pages. + * + * For columns of physical type FLOAT or DOUBLE, or logical type FLOAT16, + * NaN values are not to be included in these bounds. If all non-null values + * of a page are NaN, then a writer must do the following: + * - If the order of this column is TYPE_ORDER, then a column index must + * not be written for this column chunk. While this is unfortunate for + * performance, it is necessary to avoid conflict with legacy files that + * still included NaN in min_values and max_values even if the page had + * non-NaN values. To mitigate this, IEEE754_TOTAL_ORDER is recommended. + * - If the order of this column is IEEE754_TOTAL_ORDER, then min_values[i] + * and max_values[i] of that page must be set to the smallest and largest + * NaN values as defined by IEEE 754 total order. */ 2: required list min_values 3: required list max_values /** - * Stores whether both min_values and max_values are orderd and if so, in + * Stores whether both min_values and max_values are ordered and if so, in * which direction. This allows readers to perform binary searches in both * lists. Readers cannot assume that max_values[i] <= min_values[i+1], even * if the lists are ordered. */ 4: required BoundaryOrder boundary_order - /** A list containing the number of null values for each page **/ + /** + * A list containing the number of null values for each page + * + * Writers SHOULD always write this field even if no null values + * are present or the column is not nullable. + * Readers MUST distinguish between null_counts not being present + * and null_count being 0. + * If null_counts are not present, readers MUST NOT assume all + * null counts are 0. + */ 5: optional list null_counts + + /** + * Contains repetition level histograms for each page + * concatenated together. The repetition_level_histogram field on + * SizeStatistics contains more details. + * + * When present the length should always be (number of pages * + * (max_repetition_level + 1)) elements. + * + * Element 0 is the first element of the histogram for the first page. + * Element (max_repetition_level + 1) is the first element of the histogram + * for the second page. + **/ + 6: optional list repetition_level_histograms; + /** + * Same as repetition_level_histograms except for definitions levels. + **/ + 7: optional list definition_level_histograms; + + /** + * A list containing the number of NaN values for each page. Only present + * for columns of physical type FLOAT or DOUBLE, or logical type FLOAT16. + * If this field is not present, readers MUST assume that there might be + * NaN values in any page. + */ + 8: optional list nan_counts + } struct AesGcmV1 { @@ -988,7 +1363,14 @@ union EncryptionAlgorithm { * Description for file metadata */ struct FileMetaData { - /** Version of this file **/ + /** Version of this file + * + * As of December 2025, there is no agreed upon consensus of what constitutes + * version 2 of the file. For maximum compatibility with readers, writers should + * always populate "1" for version. For maximum compatibility with writers, + * readers should accept "1" and "2" interchangeably. All other versions are + * reserved for potential future use-cases. + */ 1: required i32 version /** Parquet schema for this file. This schema contains metadata for all the columns. @@ -1015,17 +1397,20 @@ struct FileMetaData { 6: optional string created_by /** - * Sort order used for the min_value and max_value fields of each column in - * this file. Sort orders are listed in the order matching the columns in the - * schema. The indexes are not necessary the same though, because only leaf - * nodes of the schema are represented in the list of sort orders. + * Sort order used for the min_value and max_value fields in the Statistics + * objects and the min_values and max_values fields in the ColumnIndex + * objects of each column in this file. Sort orders are listed in the order + * matching the columns in the schema. The indexes are not necessarily the same + * though, because only leaf nodes of the schema are represented in the list + * of sort orders. * - * Without column_orders, the meaning of the min_value and max_value fields is - * undefined. To ensure well-defined behaviour, if min_value and max_value are - * written to a Parquet file, column_orders must be written as well. + * Without column_orders, the meaning of the min_value and max_value fields + * in the Statistics object and the ColumnIndex object is undefined. To ensure + * well-defined behaviour, if these fields are written to a Parquet file, + * column_orders must be written as well. * - * The obsolete min and max fields are always sorted by signed comparison - * regardless of column_orders. + * The obsolete min and max fields in the Statistics object are always sorted + * by signed comparison regardless of column_orders. */ 7: optional list column_orders;