From f522bd4cf15e0b3f831bcb0287534bd152ef470b Mon Sep 17 00:00:00 2001 From: tallison Date: Wed, 28 Oct 2026 09:47:04 -0400 Subject: [PATCH 1/2] TIKA-5010: tika-grpc reads parse-context timeout-limits and closes interrupted calls with UNAVAILABLE --- CHANGES.txt | 7 ++ .../tika/pipes/grpc/TikaGrpcServerImpl.java | 47 ++++++++--- .../tika/pipes/grpc/TikaGrpcServerTest.java | 84 +++++++++++++++++++ 3 files changed, 125 insertions(+), 13 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index bef8eef13b6..e82a2e9b598 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,6 +22,13 @@ Release 4.3.0 - ??? for EKS IAM roles for service accounts (IRSA); the plugin now bundles the AWS STS module it needs (TIKA-5006). + * tika-grpc applies parse-context.timeout-limits and exception-reporting in the server + process too, so its backstop for a hung worker no longer falls back to the built-in + timeouts. A FetchAndParse call interrupted before the parse finished (server shutdown) + closes with UNAVAILABLE; a unary call used to fail with INTERNAL "no value received", + a server-streaming call completed with no reply, and a bidirectional stream went on + to throw on the closed call. Thanks to Kristian Rickert and Davide Polato (TIKA-5010). + * The Kafka emitter reports success only once Kafka has acknowledged every record (per acks); a rejected or timed-out record fails the emit instead of being lost silently, and the producer is flushed and closed on shutdown. (TIKA-5008). diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java index cc9f731460a..9735759b061 100644 --- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java @@ -118,11 +118,9 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase { TikaJsonConfig tikaJsonConfig = TikaJsonConfig.load(configPath); - // Load PipesConfig directly from root level (not from "parse-context") - pipesConfig = tikaJsonConfig.deserialize("pipes", PipesConfig.class); - if (pipesConfig == null) { - pipesConfig = new PipesConfig(); - } + // also reads parse-context.timeout-limits/exception-reporting for the client's backstop + // and the results it fabricates itself (timeouts, crashes) + pipesConfig = PipesConfig.load(tikaJsonConfig); pipesConfig.refuseTrustedRequests("tika-grpc"); // Security-sensitive grpc features (per-request config, runtime component @@ -285,25 +283,34 @@ public void fetchAndParseServerSideStreaming(FetchAndParseRequest request, if (parseContext == null) { return; } - fetchAndParseImpl(request, parseContext, responseObserver); - responseObserver.onCompleted(); + if (fetchAndParseImpl(request, parseContext, responseObserver)) { + responseObserver.onCompleted(); + } } @Override public StreamObserver fetchAndParseBiDirectionalStreaming( StreamObserver responseObserver) { return new StreamObserver<>() { + // set once the call is closed with an error; later requests have nowhere to go + private boolean closed; + @Override public void onNext(FetchAndParseRequest fetchAndParseRequest) { + if (closed) { + return; + } if (denyPerRequestConfig(fetchAndParseRequest, responseObserver)) { + closed = true; return; } ParseContext parseContext = buildRequestParseContext(fetchAndParseRequest, responseObserver); if (parseContext == null) { + closed = true; return; } - fetchAndParseImpl(fetchAndParseRequest, parseContext, responseObserver); + closed = !fetchAndParseImpl(fetchAndParseRequest, parseContext, responseObserver); } @Override @@ -313,7 +320,9 @@ public void onError(Throwable throwable) { @Override public void onCompleted() { - responseObserver.onCompleted(); + if (!closed) { + responseObserver.onCompleted(); + } } }; } @@ -328,8 +337,9 @@ public void fetchAndParse(FetchAndParseRequest request, if (parseContext == null) { return; } - fetchAndParseImpl(request, parseContext, responseObserver); - responseObserver.onCompleted(); + if (fetchAndParseImpl(request, parseContext, responseObserver)) { + responseObserver.onCompleted(); + } } /** @@ -382,8 +392,12 @@ private ParseContext buildRequestParseContext(FetchAndParseRequest request, return parseContext; } - private void fetchAndParseImpl(FetchAndParseRequest request, ParseContext parseContext, - StreamObserver responseObserver) { + /** + * Sends the reply for one request. Returns {@code false} when the call was closed with an + * error instead, so the caller must not complete it. + */ + private boolean fetchAndParseImpl(FetchAndParseRequest request, ParseContext parseContext, + StreamObserver responseObserver) { String fetcherId; try (Lease lease = fetcherManager.getFetcher(request.getFetcherId())) { fetcherId = lease.get().getExtensionConfig().id(); @@ -413,10 +427,17 @@ private void fetchAndParseImpl(FetchAndParseRequest request, ParseContext parseC } } responseObserver.onNext(fetchReplyBuilder.build()); + return true; } catch (IOException | PipesException e) { throw new RuntimeException(e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); + // no outcome to reply with; completing a unary call without a reply surfaces as INTERNAL + responseObserver.onError(io.grpc.Status.UNAVAILABLE + .withDescription("interrupted before the fetch and parse of '" + + request.getFetchKey() + "' finished") + .asRuntimeException()); + return false; } } diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java index 7dcb2d04d53..e4e28e9c551 100644 --- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java +++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java @@ -74,6 +74,8 @@ import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.pipes.api.PipesResult; +import org.apache.tika.pipes.api.fetcher.Fetcher; +import org.apache.tika.pipes.core.Lease; import org.apache.tika.pipes.grpc.proto.DeleteFetcherReply; import org.apache.tika.pipes.grpc.proto.DeleteFetcherRequest; import org.apache.tika.pipes.grpc.proto.FetchAndParseReply; @@ -795,4 +797,86 @@ public void onCompleted() { FileUtils.deleteDirectory(testDocumentFolder); } } + + @Test + public void testParentReadsOperatorTimeoutLimits() throws Exception { + // the test config sets parse-context.timeout-limits.progressTimeoutMillis + assertEquals(60000, newService(tikaConfig).pipesConfig.getDefaultTimeoutLimits() + .getProgressTimeoutMillis()); + } + + @Test + public void testInterruptedFetchAndParseClosesWithUnavailable(Resources resources) + throws Exception { + String serverName = InProcessServerBuilder.generateName(); + TikaGrpcServerImpl serviceImpl = newService(tikaConfig); + Server server = InProcessServerBuilder.forName(serverName).directExecutor() + .addService(serviceImpl).build().start(); + resources.register(server, Duration.ofSeconds(10)); + ManagedChannel channel = InProcessChannelBuilder.forName(serverName).directExecutor().build(); + resources.register(channel, Duration.ofSeconds(10)); + + String fetcherId = createFetcherId(1); + // build the fetcher up front, so only the wait for a pipes client sees the interrupt + try (Lease lease = serviceImpl.fetcherManager.getFetcher(fetcherId)) { + assertNotNull(lease.get()); + } + FetchAndParseRequest request = FetchAndParseRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetchKey("interrupted.html") + .build(); + TikaGrpc.TikaStub stub = TikaGrpc.newStub(channel); + + RecordingObserver unary = new RecordingObserver(); + RecordingObserver serverStreaming = new RecordingObserver(); + RecordingObserver biDi = new RecordingObserver(); + try { + Thread.currentThread().interrupt(); + stub.fetchAndParse(request, unary); + Thread.currentThread().interrupt(); + stub.fetchAndParseServerSideStreaming(request, serverStreaming); + // direct, not through a stub: a gRPC client drops its own sends once the call is + // closed, but requests already queued at the server still arrive + Thread.currentThread().interrupt(); + StreamObserver requests = + serviceImpl.fetchAndParseBiDirectionalStreaming(biDi); + requests.onNext(request); + Thread.interrupted(); + requests.onNext(request); + requests.onCompleted(); + } finally { + // the server restores the interrupt it caught; clear it before cleanup waits + Thread.interrupted(); + } + + for (RecordingObserver observer : List.of(unary, serverStreaming, biDi)) { + assertEquals(0, observer.replies.size(), "there is no outcome to reply with"); + assertEquals(0, observer.completions.get(), "a call closed with an error must not complete"); + assertEquals(1, observer.errors.size(), "the call must be closed with one error"); + assertEquals(Status.Code.UNAVAILABLE, + Status.fromThrowable(observer.errors.get(0)).getCode(), + String.valueOf(observer.errors.get(0))); + } + } + + private static final class RecordingObserver implements StreamObserver { + private final List replies = Collections.synchronizedList(new ArrayList<>()); + private final List errors = Collections.synchronizedList(new ArrayList<>()); + private final AtomicInteger completions = new AtomicInteger(); + + @Override + public void onNext(FetchAndParseReply reply) { + replies.add(reply); + } + + @Override + public void onError(Throwable t) { + errors.add(t); + } + + @Override + public void onCompleted() { + completions.incrementAndGet(); + } + } } From 6480d6f333656e2f693b60d6b27e5600c10a9202 Mon Sep 17 00:00:00 2001 From: tallison Date: Wed, 28 Oct 2026 10:57:13 -0400 Subject: [PATCH 2/2] TIKA-4766: v2alpha1 gRPC service, buf CI, docs, metadata type-change gate Flat /rmeta-shaped documents with registry type hints; unary, streaming and bidi with in-band per-request errors; v1 and v2 share FetchParseRuntime. buf lint/breaking CI for both protos. MetadataTypeChangesTest requires declared type changes against a committed baseline. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/buf.yml | 94 ++ .skills/devs/metadata-schema/SKILL.md | 3 + CHANGES.txt | 8 + .../ROOT/pages/using-tika/grpc/index.adoc | 81 +- pom.xml | 1 + tika-bom/pom.xml | 5 + .../apache/tika/pipes/ExternalTestBase.java | 8 +- .../tika/pipes/filesystem/V2Alpha1Test.java | 114 +++ .../src/test/resources/tika-config.json | 7 + tika-grpc-api/README.md | 19 + tika-grpc-api/buf.yaml | 28 + tika-grpc-api/pom.xml | 145 ++++ .../tika/grpc/v2alpha1/tika_service.proto | 137 +++ tika-grpc/buf.yaml | 45 + tika-grpc/pom.xml | 10 + .../tika/pipes/grpc/FetchParseRuntime.java | 141 +++ .../tika/pipes/grpc/TikaGrpcServer.java | 1 + .../tika/pipes/grpc/TikaGrpcServerImpl.java | 84 +- .../pipes/grpc/TikaV2Alpha1ServerImpl.java | 285 +++++++ .../grpc/TikaV2Alpha1ServerImplTest.java | 326 +++++++ tika-metadata-schema/README.md | 11 + .../metadata/schema/MetadataKeyValidator.java | 24 +- .../schema/MetadataTypeChangesTest.java | 133 +++ .../schema/metadata-key-types-baseline.json | 805 ++++++++++++++++++ .../tika/metadata/schema/type-changes.json | 2 + 25 files changed, 2446 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/buf.yml create mode 100644 tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/V2Alpha1Test.java create mode 100644 tika-grpc-api/README.md create mode 100644 tika-grpc-api/buf.yaml create mode 100644 tika-grpc-api/pom.xml create mode 100644 tika-grpc-api/src/main/proto/org/apache/tika/grpc/v2alpha1/tika_service.proto create mode 100644 tika-grpc/buf.yaml create mode 100644 tika-grpc/src/main/java/org/apache/tika/pipes/grpc/FetchParseRuntime.java create mode 100644 tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImpl.java create mode 100644 tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImplTest.java create mode 100644 tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataTypeChangesTest.java create mode 100644 tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/metadata-key-types-baseline.json create mode 100644 tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/type-changes.json diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml new file mode 100644 index 00000000000..f1fb7c0d52a --- /dev/null +++ b/.github/workflows/buf.yml @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: buf lint and breaking + +on: + pull_request: + branches: [ main ] + paths: + - '**/*.proto' + - '**/buf.yaml' + - '.github/workflows/buf.yml' + push: + branches: [ main ] + paths: + - '**/*.proto' + - '**/buf.yaml' + - '.github/workflows/buf.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + BUF_VERSION: 1.50.0 + # sha256 of buf-Linux-x86_64 from the release's sha256.txt; a download that does not + # match fails the job instead of running an unverified binary. + BUF_SHA256: 154ea883ce098eac4fa106ff9ee4e4964bb97f809dd8ec9c34a432b466ce1494 + +jobs: + buf: + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + # tika-grpc holds the frozen v1 tika.proto, tika-grpc-api the v2alpha1 protos. + module: [ tika-grpc, tika-grpc-api ] + steps: + # actions/checkout v7.0.1, pinned by commit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + - name: install buf + run: | + curl -sSfL -o "$RUNNER_TEMP/buf" \ + "https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/buf-Linux-x86_64" + echo "${BUF_SHA256} $RUNNER_TEMP/buf" | sha256sum -c - + chmod +x "$RUNNER_TEMP/buf" + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + - name: buf lint + working-directory: ${{ matrix.module }} + run: buf lint + # Pull requests compare with the base branch; pushes with the commit before the push. + # The rules come from the base's buf.yaml, so a change cannot loosen the check that + # judges it. + - name: buf breaking + env: + BASE: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.before }} + MODULE: ${{ matrix.module }} + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch --no-tags origin "$BASE" + BASE_SHA=$(git rev-parse FETCH_HEAD) + else + BASE_SHA="$BASE" + fi + if ! git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null \ + || [ -z "$(git ls-tree -r --name-only "$BASE_SHA" -- "$MODULE/src/main/proto" | grep '\.proto$')" ]; then + echo "$BASE has no protos under $MODULE yet; nothing to compare." + exit 0 + fi + if git cat-file -e "$BASE_SHA:$MODULE/buf.yaml" 2>/dev/null; then + git show "$BASE_SHA:$MODULE/buf.yaml" > "$MODULE/buf.yaml" + fi + buf breaking "$MODULE" \ + --against ".git#ref=$BASE_SHA,subdir=$MODULE/src/main/proto" diff --git a/.skills/devs/metadata-schema/SKILL.md b/.skills/devs/metadata-schema/SKILL.md index 68f126a3cae..39c5f6fef94 100644 --- a/.skills/devs/metadata-schema/SKILL.md +++ b/.skills/devs/metadata-schema/SKILL.md @@ -88,6 +88,9 @@ under the full test classpath and asserts a byte-match — the definitive comple reflects the *registry*, not live code — regenerate before trusting it. - `MetadataKeyValidatorTest` — registry-driven CLOSED/OPEN/TEMPLATE/UNKNOWN classifier. - `MetadataCoverageTest` — fails if a scanned module's keys are neither in scope nor listed out-of-scope. +- `MetadataTypeChangesTest` — a key's valueType/cardinality may differ from the committed baseline only + if `type-changes.json` declares it (key, from, to, jira). Clients consume these types (gRPC v2 hints, + typed /rmeta JSON), so a change also needs a CHANGES row. Refresh the baseline at release (README). Failures with stale `X-TIKA:`/underscore/`SHA256` keys usually mean *regenerate*, not edit code. diff --git a/CHANGES.txt b/CHANGES.txt index e82a2e9b598..a2fe96d1bdf 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,6 +29,14 @@ Release 4.3.0 - ??? a server-streaming call completed with no reply, and a bidirectional stream went on to throw on the closed call. Thanks to Kristian Rickert and Davide Polato (TIKA-5010). + * tika-grpc adds an experimental service, org.apache.tika.grpc.v2alpha1.TikaService, on + the same port as v1. It returns what /rmeta returns: one document per container and + embedded file, each with its content and every metadata value (v1 keeps one value per + key and merges embedded files into one map). Values are strings, with type and + cardinality hints from tika-metadata-schema. Protos are in the new tika-grpc-api + artifact. The package may change incompatibly until it becomes v2. Thanks to Kristian + Rickert and Davide Polato for the design work in #2961 (TIKA-4766). + * The Kafka emitter reports success only once Kafka has acknowledged every record (per acks); a rejected or timed-out record fails the emit instead of being lost silently, and the producer is flushed and closed on shutdown. (TIKA-5008). diff --git a/docs/modules/ROOT/pages/using-tika/grpc/index.adoc b/docs/modules/ROOT/pages/using-tika/grpc/index.adoc index 0642be675f7..ac72fdc21cc 100644 --- a/docs/modules/ROOT/pages/using-tika/grpc/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/grpc/index.adoc @@ -25,7 +25,8 @@ The service definition lives in `tika-grpc/src/main/proto/tika.proto`. Clients r (`SaveFetcher`) and then submit `FetchAndParseRequest` messages, each returning a `FetchAndParseReply` with extracted metadata and content. Streaming variants (`FetchAndParseServerSideStreaming`, `FetchAndParseBiDirectionalStreaming`) return the same reply -type. +type. An experimental second service returns every document and every metadata value; see +<<_v2alpha1,The v2alpha1 service>>. == Concurrency @@ -33,7 +34,83 @@ type. `pipes.numClients` (default: derived from host cores, at most 4). A call that cannot get a fork within `pipes.maxWaitForClientMillis` (default 60s) returns the in-band reply status `CLIENT_UNAVAILABLE_WITHIN_MS` — at capacity, not failing. `pipes.useSharedServer: true` shares one -JVM instead. +JVM instead. (The v2alpha1 service answers that case with `RESOURCE_EXHAUSTED`.) + +== Message size + +gRPC clients reject replies larger than 4 MB by default, and a large document's text easily +exceeds that. Raise the limit on the client channel, up to the server's ceiling, +`pipes.maxIpcPayloadBytes` (100 MB by default): + +* Java: `ManagedChannelBuilder.forTarget(...).maxInboundMessageSize(100 * 1024 * 1024)` +* Python: `grpc.insecure_channel(target, options=[("grpc.max_receive_message_length", 100 * 1024 * 1024)])` +* Go: `grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(100 << 20))` +* grpcurl: `-max-msg-sz 104857600` + +gRPC compression saves bandwidth but does not help with this limit, which applies to the +decompressed size. `writeLimit` in the parse context caps how much text is extracted. The +v2alpha1 `FetchAndParseStream` RPC sends one message per document, so an archive's attachments +each count against the limit separately. + +[#_v2alpha1] +== The v2alpha1 service (experimental) + +`org.apache.tika.grpc.v2alpha1.TikaService` runs on the same port as the v1 service. Its protos +are in the `tika-grpc-api` artifact, so clients can depend on them without the server. The +package may change incompatibly until it is renamed to `v2`. + +It returns what `/rmeta` returns. Each reply holds one `ParsedDocument` for the container and one +for each embedded file, and each `ParsedDocument` carries: + +* `embedded_path`: empty for the container, otherwise the file's `tk:embedded-id-path`. +* `content`: the extracted text, in the format named by `tk:content-handler-type`. +* `metadata`: every key with all of its values. v1's `fields` map keeps one value per key and +merges embedded files into one map. + +Metadata values are always strings, exactly as Tika stores them. Each key may also carry a type +hint (`VALUE_TYPE_INTEGER`, `VALUE_TYPE_DATE`, ...) and a cardinality hint (`CARDINALITY_BAG`, +`CARDINALITY_SEQ`, ...). Hints come from Tika's metadata key registry (`tika-metadata-schema`), and +an absent hint means text. Hints can change between Tika versions, so treat them this way: + +* Read them on every response. +* Fall back to the string when a value does not parse as its hinted type. +* Never hard-code a key's type. + +Dates are written as `yyyy-MM-ddTHH:mm:ssZ` (UTC), `yyyy-MM-ddTHH:mm:ss` (no time zone was +known), or `yyyy-MM-dd`. + +[cols="1,3",options="header"] +|=== +|RPC |Behavior + +|`FetchAndParse` +|One `FetchAndParseResponse` with every document. + +|`FetchAndParseStream` +|One message per document, in the same order. + +|`FetchAndParseBidi` +|Many requests on one stream, one response per request, matched by `request_id`. A request that +fails gets a response with `error` set, and the stream continues. +|=== + +Failures that are not about the document close a unary or streaming call with a gRPC status: + +* `NOT_FOUND`: unknown fetcher. +* `RESOURCE_EXHAUSTED`: no worker became free in time. +* `UNAVAILABLE`: a worker or fetcher failed to start, or the server is shutting down. +* `PERMISSION_DENIED`: per-request config was sent but is not allowed. +* `INVALID_ARGUMENT`: bad `parse_context_json`. + +On `FetchAndParseBidi`, the same codes arrive in that request's `error` instead, and only server +shutdown ends the stream. Everything about the document itself is a normal reply: `pipes_status` +says what happened (`PARSE_SUCCESS`, `FETCH_EXCEPTION`, `TIMEOUT`, ...) and `message` says why. Each +document's `tk:exception:*` metadata holds exceptions the parse recorded. How much exception detail +callers see is set by +xref:advanced/setting-limits.adoc#_exception_reporting[`parse-context.exception-reporting`]. + +The v2alpha1 service goes through the same capability flags and request checks as v1. Fetchers are +still managed through v1 (`SaveFetcher` and related RPCs). == Command-line options diff --git a/pom.xml b/pom.xml index 12a26f6059f..7d88b2b1bcf 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,7 @@ tika-langdetect tika-pipes + tika-grpc-api tika-grpc tika-app tika-server diff --git a/tika-bom/pom.xml b/tika-bom/pom.xml index 308b0c7cb6a..bc0fb7a468e 100644 --- a/tika-bom/pom.xml +++ b/tika-bom/pom.xml @@ -387,6 +387,11 @@ tika-async-cli ${revision} + + org.apache.tika + tika-grpc-api + ${revision} + org.apache.tika tika-httpclient-commons diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java index 5ebe3d569dd..f86a7f339e2 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java @@ -89,14 +89,16 @@ static void setup() throws Exception { } /** - * The checked-in configs name the Docker image's plugins dir; the local server gets a copy - * pointed at the zip this module copies into target/plugins. + * The checked-in configs name the Docker image's plugins and fixtures dirs; the local server + * gets a copy pointed at target/plugins and the fixtures folder. */ public static Path localConfig(String name) throws IOException { Path pluginsDir = Path.of("target/plugins").toAbsolutePath(); String json = Files.readString(Path.of("src/test/resources", name)); String local = json.replace("/var/cache/tika/plugins", - pluginsDir.toString().replace("\\", "\\\\")); + pluginsDir.toString().replace("\\", "\\\\")) + .replace(GOV_DOCS_FOLDER, + TEST_FOLDER.getAbsolutePath().replace("\\", "\\\\")); Path out = Path.of("target", name).toAbsolutePath(); Files.writeString(out, local); return out; diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/V2Alpha1Test.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/V2Alpha1Test.java new file mode 100644 index 00000000000..2c8f9031258 --- /dev/null +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/V2Alpha1Test.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.filesystem; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.apache.tika.grpc.v2alpha1.Cardinality; +import org.apache.tika.grpc.v2alpha1.FetchAndParseBidiRequest; +import org.apache.tika.grpc.v2alpha1.FetchAndParseBidiResponse; +import org.apache.tika.grpc.v2alpha1.FetchAndParseRequest; +import org.apache.tika.grpc.v2alpha1.FetchAndParseResponse; +import org.apache.tika.grpc.v2alpha1.ParsedDocument; +import org.apache.tika.grpc.v2alpha1.TikaServiceGrpc; +import org.apache.tika.grpc.v2alpha1.Values; +import org.apache.tika.pipes.ExternalTestBase; + +/** The v2alpha1 service on a live server, next to v1 on the same port. */ +@DisabledOnOs(value = OS.WINDOWS, disabledReason = "exec:exec classpath exceeds Windows CreateProcess command-line length limit") +class V2Alpha1Test extends ExternalTestBase { + + // declared in tika-config.json: a fetcher saved at runtime would not reach the forked + // workers under the default in-memory config store + private static final String FETCHER_ID = "fixtures"; + + @Test + void v2ParsesAndReportsFailuresInBand() throws Exception { + ManagedChannel channel = getManagedChannel(); + try { + FetchAndParseResponse csv = TikaServiceGrpc.newBlockingStub(channel) + .fetchAndParse(FetchAndParseRequest.newBuilder() + .setFetcherId(FETCHER_ID).setFetchKey("sample.csv").build()); + assertEquals("PARSE_SUCCESS", csv.getPipesStatus(), csv.getMessage()); + ParsedDocument document = csv.getDocuments(0); + assertTrue(!document.getContent().isBlank(), "content"); + assertTrue(document.containsMetadata("tk:content-handler-type")); + // a registry hint on the wire: every parser in the chain, as a BAG + Values parsedBy = document.getMetadataOrThrow("tk:parsed-by"); + assertEquals(Cardinality.CARDINALITY_BAG, parsedBy.getCardinality()); + assertTrue(parsedBy.getValuesCount() > 1, parsedBy.toString()); + + List responses = Collections.synchronizedList(new ArrayList<>()); + List errors = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch done = new CountDownLatch(1); + StreamObserver requests = TikaServiceGrpc.newStub(channel) + .fetchAndParseBidi(new StreamObserver<>() { + @Override + public void onNext(FetchAndParseBidiResponse response) { + responses.add(response); + } + + @Override + public void onError(Throwable t) { + errors.add(t); + done.countDown(); + } + + @Override + public void onCompleted() { + done.countDown(); + } + }); + requests.onNext(bidi("html", FETCHER_ID, "sample.html")); + requests.onNext(bidi("missing-fetcher", "no-such-fetcher", "sample.html")); + requests.onNext(bidi("xml", FETCHER_ID, "sample.xml")); + requests.onCompleted(); + assertTrue(done.await(2, TimeUnit.MINUTES), "bidi stream did not finish"); + + assertTrue(errors.isEmpty(), String.valueOf(errors)); + Map byId = new HashMap<>(); + responses.forEach(r -> byId.put(r.getRequestId(), r)); + assertEquals(3, byId.size()); + assertEquals("PARSE_SUCCESS", byId.get("html").getResult().getPipesStatus()); + assertEquals(Status.Code.NOT_FOUND.value(), byId.get("missing-fetcher").getError().getCode()); + assertEquals("PARSE_SUCCESS", byId.get("xml").getResult().getPipesStatus()); + } finally { + channel.shutdown(); + } + } + + private static FetchAndParseBidiRequest bidi(String id, String fetcherId, String fetchKey) { + return FetchAndParseBidiRequest.newBuilder() + .setRequestId(id).setFetcherId(fetcherId).setFetchKey(fetchKey).build(); + } +} diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json index 0529991d3d3..8adbff5d029 100644 --- a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json +++ b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json @@ -31,6 +31,13 @@ "--add-opens=java.desktop/java.awt.font=ALL-UNNAMED" ] }, + "fetchers": { + "fixtures": { + "file-system-fetcher": { + "basePath": "/tika/govdocs1" + } + } + }, "grpc": { "allowComponentManagement": true } diff --git a/tika-grpc-api/README.md b/tika-grpc-api/README.md new file mode 100644 index 00000000000..bf7c79ef11c --- /dev/null +++ b/tika-grpc-api/README.md @@ -0,0 +1,19 @@ +# tika-grpc-api + +Protobuf definitions and generated Java stubs for the experimental +`org.apache.tika.grpc.v2alpha1.TikaService`, served by `tika-grpc` next to the v1 service. Depend +on this artifact to build a client without pulling in the server. + +The package may change incompatibly until it is renamed to `org.apache.tika.grpc.v2`. The +generated `FileDescriptorSet` is in `META-INF/org.apache.tika.grpc.v2alpha1.descriptors` for +clients that work from descriptors. + +User documentation, covering the reply shape, type hints, errors and client message size, is the +"v2alpha1 service" section of the Tika gRPC page in the Tika docs +(`docs/modules/ROOT/pages/using-tika/grpc/index.adoc`). + +Lint and breaking-change checks: + +```bash +cd tika-grpc-api && buf lint +``` diff --git a/tika-grpc-api/buf.yaml b/tika-grpc-api/buf.yaml new file mode 100644 index 00000000000..eaaffab6bc4 --- /dev/null +++ b/tika-grpc-api/buf.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: v2 +modules: + - path: src/main/proto +lint: + use: + - STANDARD +breaking: + use: + - FILE + # v2alpha1 may change incompatibly; the check applies once the package is renamed to v2 + ignore_unstable_packages: true diff --git a/tika-grpc-api/pom.xml b/tika-grpc-api/pom.xml new file mode 100644 index 00000000000..447881840f3 --- /dev/null +++ b/tika-grpc-api/pom.xml @@ -0,0 +1,145 @@ + + + + 4.0.0 + tika-grpc-api + jar + Apache Tika gRPC API + https://tika.apache.org/ + + + org.apache.tika + tika-parent + ${revision} + ../tika-parent/pom.xml + + + + + com.google.protobuf + protobuf-java + + + io.grpc + grpc-protobuf + + + com.google.errorprone + error_prone_annotations + + + + + io.grpc + grpc-stub + + + com.google.errorprone + error_prone_annotations + + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + ${project.basedir}/src/main/proto + + org/apache/tika/grpc/v2alpha1/*.proto + + true + true + ${project.build.outputDirectory}/META-INF + org.apache.tika.grpc.v2alpha1.descriptors + + + + + compile + compile-custom + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.1 + + + add-protobuf-sources + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources/protobuf/java + ${project.build.directory}/generated-sources/protobuf/grpc-java + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.tika.grpc.api + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + org.apache.rat + apache-rat-plugin + + + README.md + + + + + + diff --git a/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v2alpha1/tika_service.proto b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v2alpha1/tika_service.proto new file mode 100644 index 00000000000..972c9c5ec7b --- /dev/null +++ b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v2alpha1/tika_service.proto @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +// Experimental: this package may change incompatibly until it is renamed to +// org.apache.tika.grpc.v2. +package org.apache.tika.grpc.v2alpha1; + +option java_multiple_files = true; +option java_package = "org.apache.tika.grpc.v2alpha1"; +option java_outer_classname = "TikaServiceProto"; + +// What Tika produces for one fetch and parse, as /rmeta returns it: one ParsedDocument +// for the container and one for each embedded document. +service TikaService { + rpc FetchAndParse(FetchAndParseRequest) returns (FetchAndParseResponse); + // One document per message, in the order of FetchAndParseResponse.documents. + rpc FetchAndParseStream(FetchAndParseStreamRequest) returns (stream FetchAndParseStreamResponse); + // One response per request. A request that fails gets a response with `error` set; the + // stream continues. Match responses to requests by request_id, not by order. + rpc FetchAndParseBidi(stream FetchAndParseBidiRequest) returns (stream FetchAndParseBidiResponse); +} + +message FetchAndParseRequest { + string fetcher_id = 1; + string fetch_key = 2; + // Refused unless the operator allows per-request config, as in v1. + string parse_context_json = 3; + string additional_fetch_config_json = 4; +} + +message FetchAndParseStreamRequest { + string fetcher_id = 1; + string fetch_key = 2; + string parse_context_json = 3; + string additional_fetch_config_json = 4; +} + +message FetchAndParseBidiRequest { + string fetcher_id = 1; + string fetch_key = 2; + string parse_context_json = 3; + string additional_fetch_config_json = 4; + // Chosen by the client and echoed in the response. + string request_id = 5; +} + +message FetchAndParseResponse { + string fetch_key = 1; + // The pipes result status (PARSE_SUCCESS, PARSE_SUCCESS_WITH_EXCEPTION, TIMEOUT, ...). A string, + // so new statuses need no proto change. + string pipes_status = 2; + // [0] is the container; embedded documents follow in the order Tika reports them. + repeated ParsedDocument documents = 3; + // Why a fetch, parse or worker failed (fetch exception, timeout, crash), formatted under the + // operator's parse-context.exception-reporting. Exceptions the parse recorded alongside its + // output are in each document's tk:exception:* metadata instead. + string message = 4; +} + +message FetchAndParseStreamResponse { + ParsedDocument document = 1; +} + +message FetchAndParseBidiResponse { + string request_id = 1; + // Set when the request failed for a reason that is not about the document. + RequestError error = 2; + // Set otherwise. + FetchAndParseResponse result = 3; +} + +message RequestError { + // The gRPC status code a unary call would have closed with (5 NOT_FOUND, 7 PERMISSION_DENIED, + // 8 RESOURCE_EXHAUSTED, 14 UNAVAILABLE, ...). + int32 code = 1; + string message = 2; +} + +message ParsedDocument { + // Empty for the container; otherwise the document's tk:embedded-id-path. + string embedded_path = 1; + // The extracted content. Its format is in metadata under tk:content-handler-type (TEXT, HTML, + // XML, MARKDOWN, or a custom handler's own name). Not repeated in metadata. + string content = 2; + // Every metadata key and all of its values. + map metadata = 3; +} + +message Values { + // Tika's strings, in Tika's order. Always present; the hints below never replace them. + repeated string values = 1; + // Hints from Tika's metadata key registry. They can change between Tika versions: read them + // on every response, fall back to the strings when a value does not parse, and never + // hard-code a key's type. + ValueType type = 2; + Cardinality cardinality = 3; +} + +// The string form for each type is fixed: BOOLEAN true/false; INTEGER decimal; REAL a Java +// double; RATIONAL n/d; DATE yyyy-MM-ddTHH:mm:ssZ (UTC), yyyy-MM-ddTHH:mm:ss (no zone known) +// or yyyy-MM-dd; JSON a JSON document. Treat a value you do not recognize as text. +enum ValueType { + // Text, or no declared type. + VALUE_TYPE_UNSPECIFIED = 0; + VALUE_TYPE_BOOLEAN = 1; + VALUE_TYPE_INTEGER = 2; + VALUE_TYPE_REAL = 3; + VALUE_TYPE_RATIONAL = 4; + VALUE_TYPE_DATE = 5; + VALUE_TYPE_URI = 6; + VALUE_TYPE_JSON = 7; +} + +enum Cardinality { + // A single value, or no declared cardinality. + CARDINALITY_UNSPECIFIED = 0; + // Unordered values. + CARDINALITY_BAG = 1; + // Ordered values; position is meaningful (pdf:charsPerPage). + CARDINALITY_SEQ = 2; + // Alternatives. + CARDINALITY_ALT = 3; +} diff --git a/tika-grpc/buf.yaml b/tika-grpc/buf.yaml new file mode 100644 index 00000000000..432a200be5e --- /dev/null +++ b/tika-grpc/buf.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The v1 tika.Tika service. Its package, service and message names are the wire API +# v1 clients already call, so the STANDARD rules that would rename them are excepted; +# every other STANDARD rule applies. Behaviour changes go to the v2 service +# (tika-grpc-api), which passes STANDARD with no exceptions. The breaking check keeps +# v1 from drifting: +# +# cd tika-grpc && buf lint +# buf breaking tika-grpc --against '.git#branch=main,subdir=tika-grpc/src/main/proto' +version: v2 +modules: + - path: src/main/proto +lint: + use: + - STANDARD + except: + # package tika, at the root of src/main/proto + - PACKAGE_DIRECTORY_MATCH + - PACKAGE_VERSION_SUFFIX + # service Tika + - SERVICE_SUFFIX + # FooRequest/FooReply, and FetchAndParseRequest/Reply shared by the three + # FetchAndParse RPCs + - RPC_REQUEST_STANDARD_NAME + - RPC_RESPONSE_STANDARD_NAME + - RPC_REQUEST_RESPONSE_UNIQUE +breaking: + use: + - FILE diff --git a/tika-grpc/pom.xml b/tika-grpc/pom.xml index ce2fa7c4389..63e16cfb590 100644 --- a/tika-grpc/pom.xml +++ b/tika-grpc/pom.xml @@ -133,6 +133,16 @@ j2objc-annotations ${j2objc-annotations.version} + + ${project.groupId} + tika-grpc-api + ${project.version} + + + ${project.groupId} + tika-metadata-schema + ${project.version} + org.apache.tika tika-async-cli diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/FetchParseRuntime.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/FetchParseRuntime.java new file mode 100644 index 00000000000..157087a4b52 --- /dev/null +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/FetchParseRuntime.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.grpc; + +import java.io.IOException; + +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import io.grpc.Status; +import io.grpc.StatusException; +import org.apache.commons.lang3.StringUtils; + +import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.pipes.api.FetchEmitTuple; +import org.apache.tika.pipes.api.PipesResult; +import org.apache.tika.pipes.api.emitter.EmitKey; +import org.apache.tika.pipes.api.fetcher.FetchKey; +import org.apache.tika.pipes.api.fetcher.Fetcher; +import org.apache.tika.pipes.core.Lease; +import org.apache.tika.pipes.core.PipesException; +import org.apache.tika.pipes.core.PipesParser; +import org.apache.tika.pipes.core.fetcher.FetcherManager; +import org.apache.tika.serialization.ComponentNameResolver; +import org.apache.tika.serialization.ParseContextUtils; +import org.apache.tika.serialization.serdes.ParseContextDeserializer; + +/** + * The fetch-and-parse path every gRPC service version shares: the per-request config gates, fetcher + * lookup and the pipes call. Holds no state of its own; the v1 server builds and closes what it + * uses. + */ +final class FetchParseRuntime { + + // duplicate keys in parse_context_json are refused, not last-one-wins + private static final ObjectMapper JSON = JsonMapper.builder() + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build(); + + private final TikaGrpcConfig tikaGrpcConfig; + private final FetcherManager fetcherManager; + private final PipesParser pipesParser; + + FetchParseRuntime(TikaGrpcConfig tikaGrpcConfig, FetcherManager fetcherManager, + PipesParser pipesParser) { + this.tikaGrpcConfig = tikaGrpcConfig; + this.fetcherManager = fetcherManager; + this.pipesParser = pipesParser; + } + + /** {@code PERMISSION_DENIED} if the request carries per-request config the operator has not + * allowed, else {@code null}. */ + Status perRequestConfigDenial(String additionalFetchConfigJson, String parseContextJson) { + boolean hasPerRequestConfig = StringUtils.isNotBlank(additionalFetchConfigJson) + || StringUtils.isNotBlank(parseContextJson); + if (!hasPerRequestConfig || tikaGrpcConfig.isAllowPerRequestConfig()) { + return null; + } + return Status.PERMISSION_DENIED + .withDescription("Per-request configuration is disabled. Set " + + "'allowPerRequestConfig' to true in the 'grpc' section of your " + + "tika-config to allow additional_fetch_config_json / " + + "parse_context_json. Understand the security implications first."); + } + + /** + * Builds the per-request {@link ParseContext} from {@code parse_context_json}, or throws + * {@code INVALID_ARGUMENT}. + *

+ * The request is untrusted wire input, so it is deserialized restricted and resolved here, as + * the tika-server resources do. Left to the fork, the same refusal (of + * {@code exception-reporting}, say) is a deserialization failure that exits it: a JVM restart + * per bad request, and a crash status with no reason for the caller. The trade-off, accepted + * deliberately: a component registered only on the fork's classpath is refused here too -- + * fail-fast validation runs against this JVM's registries. Serialization still forwards the + * caller's original jsonConfigs, not the instances resolution materializes. + */ + ParseContext requestParseContext(String fetcherId, String parseContextJson, + String additionalFetchConfigJson) throws StatusException { + ParseContext parseContext = new ParseContext(); + if (StringUtils.isNotBlank(parseContextJson)) { + try { + parseContext = ParseContextDeserializer.readParseContext( + JSON.readTree(parseContextJson), true); + ParseContextUtils.resolveAll(parseContext, getClass().getClassLoader()); + } catch (IOException | TikaConfigException e) { + throw Status.INVALID_ARGUMENT + .withDescription("Invalid parse_context_json: " + e.getMessage()) + .asException(); + } + } + if (StringUtils.isNotBlank(additionalFetchConfigJson)) { + // The fork reads this jsonConfig by the fetcher's registered component name + // (e.g. "http-fetcher"). An unregistered key fails the fork's resolveAll and a + // wire-blocked one is refused by its restricted tuple deserialization -- both + // exit the worker. Enforce here: a 400, not a JVM restart per request. + var info = ComponentNameResolver.getComponentInfo(fetcherId); + if (info.isEmpty() || ComponentNameResolver.isWireBlocked( + ComponentNameResolver.determineContextKey(info.get()))) { + throw Status.INVALID_ARGUMENT + .withDescription("additional_fetch_config_json requires fetcher_id to " + + "be a registered, wire-allowed component name; got '" + + fetcherId + "'") + .asException(); + } + parseContext.setJsonConfig(fetcherId, additionalFetchConfigJson); + } + return parseContext; + } + + /** The registered id of {@code fetcherId}. */ + String resolveFetcherId(String fetcherId) throws TikaException, IOException { + try (Lease lease = fetcherManager.getFetcher(fetcherId)) { + return lease.get().getExtensionConfig().id(); + } + } + + /** One fetch and parse through pipes, emitting nothing. */ + PipesResult fetchAndParse(String resolvedFetcherId, String fetchKey, ParseContext parseContext) + throws IOException, PipesException, InterruptedException { + return pipesParser.parse(new FetchEmitTuple(fetchKey, new FetchKey(resolvedFetcherId, fetchKey), + new EmitKey(), new Metadata(), parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP)); + } +} diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java index 3a98ba6c428..0815779eccd 100644 --- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java @@ -113,6 +113,7 @@ public void start() throws Exception { server = Grpc .newServerBuilderForPort(port, creds) .addService(serviceImpl) + .addService(new TikaV2Alpha1ServerImpl(serviceImpl.runtime)) .addService(healthStatusManager.getHealthService()) .addService(ProtoReflectionServiceV1.newInstance()) .build() diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java index 9735759b061..f8d984e388c 100644 --- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java @@ -31,9 +31,9 @@ import com.fasterxml.jackson.module.jsonSchema.JsonSchema; import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator; import com.google.rpc.Status; +import io.grpc.StatusException; import io.grpc.protobuf.StatusProto; import io.grpc.stub.StreamObserver; -import org.apache.commons.lang3.StringUtils; import org.pf4j.PluginManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,10 +43,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.ParseContext; -import org.apache.tika.pipes.api.FetchEmitTuple; import org.apache.tika.pipes.api.PipesResult; -import org.apache.tika.pipes.api.emitter.EmitKey; -import org.apache.tika.pipes.api.fetcher.FetchKey; import org.apache.tika.pipes.api.fetcher.Fetcher; import org.apache.tika.pipes.api.fetcher.FetcherFactory; import org.apache.tika.pipes.api.pipesiterator.PipesIteratorFactory; @@ -79,9 +76,6 @@ import org.apache.tika.pipes.grpc.proto.TikaGrpc; import org.apache.tika.plugins.ExtensionConfig; import org.apache.tika.plugins.TikaPluginManager; -import org.apache.tika.serialization.ComponentNameResolver; -import org.apache.tika.serialization.ParseContextUtils; -import org.apache.tika.serialization.serdes.ParseContextDeserializer; class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase { private static final Logger LOG = LoggerFactory.getLogger(TikaGrpcServerImpl.class); @@ -101,6 +95,7 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase { ConfigStore configStore; Path tikaConfigPath; PluginManager pluginManager; + FetchParseRuntime runtime; private AutoCloseable igniteStoreServer; TikaGrpcServerImpl(String tikaConfigPath) throws TikaConfigException, IOException { @@ -170,6 +165,7 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase { fetcherManager = FetcherManager.load(pluginManager, tikaJsonConfig, tikaGrpcConfig.isAllowComponentManagement(), this.configStore); + runtime = new FetchParseRuntime(tikaGrpcConfig, fetcherManager, pipesParser); } private ConfigStore createConfigStore() throws TikaConfigException { @@ -258,18 +254,12 @@ private boolean denyComponentManagement(StreamObserver responseObserver) { */ private boolean denyPerRequestConfig(FetchAndParseRequest request, StreamObserver responseObserver) { - boolean hasPerRequestConfig = - StringUtils.isNotBlank(request.getAdditionalFetchConfigJson()) - || StringUtils.isNotBlank(request.getParseContextJson()); - if (!hasPerRequestConfig || tikaGrpcConfig.isAllowPerRequestConfig()) { + io.grpc.Status denial = runtime.perRequestConfigDenial( + request.getAdditionalFetchConfigJson(), request.getParseContextJson()); + if (denial == null) { return false; } - responseObserver.onError(io.grpc.Status.PERMISSION_DENIED - .withDescription("Per-request configuration is disabled. Set " - + "'allowPerRequestConfig' to true in the 'grpc' section of your " - + "tika-config to allow additional_fetch_config_json / " - + "parse_context_json. Understand the security implications first.") - .asRuntimeException()); + responseObserver.onError(denial.asRuntimeException()); return true; } @@ -343,53 +333,19 @@ public void fetchAndParse(FetchAndParseRequest request, } /** - * Builds the per-request {@link ParseContext} from {@code parse_context_json}, or closes the - * call with {@code INVALID_ARGUMENT} and returns {@code null}. The caller must {@code return} - * immediately on {@code null}. - *

- * The request is untrusted wire input, so it is deserialized restricted and resolved here, as - * the tika-server resources do. Left to the fork, the same refusal (of - * {@code exception-reporting}, say) is a deserialization failure that exits it: a JVM restart - * per bad request, and a crash status with no reason for the caller. The trade-off, accepted - * deliberately: a component registered only on the fork's classpath is refused here too -- - * fail-fast validation runs against this JVM's registries. Serialization still forwards the - * caller's original jsonConfigs, not the instances resolution materializes. + * See {@link FetchParseRuntime#requestParseContext}; closes the call with + * {@code INVALID_ARGUMENT} and returns {@code null} instead of throwing. The caller must + * {@code return} immediately on {@code null}. */ private ParseContext buildRequestParseContext(FetchAndParseRequest request, StreamObserver responseObserver) { - ParseContext parseContext = new ParseContext(); - String parseContextJson = request.getParseContextJson(); - if (StringUtils.isNotBlank(parseContextJson)) { - try { - parseContext = ParseContextDeserializer.readParseContext( - OBJECT_MAPPER.readTree(parseContextJson), true); - ParseContextUtils.resolveAll(parseContext, getClass().getClassLoader()); - } catch (IOException | TikaConfigException e) { - responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT - .withDescription("Invalid parse_context_json: " + e.getMessage()) - .asRuntimeException()); - return null; - } - } - String additionalFetchConfigJson = request.getAdditionalFetchConfigJson(); - if (StringUtils.isNotBlank(additionalFetchConfigJson)) { - // The fork reads this jsonConfig by the fetcher's registered component name - // (e.g. "http-fetcher"). An unregistered key fails the fork's resolveAll and a - // wire-blocked one is refused by its restricted tuple deserialization -- both - // exit the worker. Enforce here: a 400, not a JVM restart per request. - var info = ComponentNameResolver.getComponentInfo(request.getFetcherId()); - if (info.isEmpty() || ComponentNameResolver.isWireBlocked( - ComponentNameResolver.determineContextKey(info.get()))) { - responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT - .withDescription("additional_fetch_config_json requires fetcher_id to " - + "be a registered, wire-allowed component name; got '" - + request.getFetcherId() + "'") - .asRuntimeException()); - return null; - } - parseContext.setJsonConfig(request.getFetcherId(), additionalFetchConfigJson); + try { + return runtime.requestParseContext(request.getFetcherId(), request.getParseContextJson(), + request.getAdditionalFetchConfigJson()); + } catch (StatusException e) { + responseObserver.onError(e); + return null; } - return parseContext; } /** @@ -399,16 +355,14 @@ private ParseContext buildRequestParseContext(FetchAndParseRequest request, private boolean fetchAndParseImpl(FetchAndParseRequest request, ParseContext parseContext, StreamObserver responseObserver) { String fetcherId; - try (Lease lease = fetcherManager.getFetcher(request.getFetcherId())) { - fetcherId = lease.get().getExtensionConfig().id(); + try { + fetcherId = runtime.resolveFetcherId(request.getFetcherId()); } catch (TikaException | IOException e) { throw new RuntimeException("Could not find fetcher with name " + request.getFetcherId(), e); } - Metadata tikaMetadata = new Metadata(); try { - PipesResult pipesResult = pipesParser.parse(new FetchEmitTuple(request.getFetchKey(), new FetchKey(fetcherId, request.getFetchKey()), - new EmitKey(), tikaMetadata, parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP)); + PipesResult pipesResult = runtime.fetchAndParse(fetcherId, request.getFetchKey(), parseContext); FetchAndParseReply.Builder fetchReplyBuilder = FetchAndParseReply.newBuilder() .setFetchKey(request.getFetchKey()) diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImpl.java new file mode 100644 index 00000000000..6a2ba3dce22 --- /dev/null +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImpl.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.grpc; + +import java.io.IOException; +import java.util.Arrays; + +import io.grpc.Status; +import io.grpc.StatusException; +import io.grpc.stub.StreamObserver; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.grpc.v2alpha1.Cardinality; +import org.apache.tika.grpc.v2alpha1.FetchAndParseBidiRequest; +import org.apache.tika.grpc.v2alpha1.FetchAndParseBidiResponse; +import org.apache.tika.grpc.v2alpha1.FetchAndParseRequest; +import org.apache.tika.grpc.v2alpha1.FetchAndParseResponse; +import org.apache.tika.grpc.v2alpha1.FetchAndParseStreamRequest; +import org.apache.tika.grpc.v2alpha1.FetchAndParseStreamResponse; +import org.apache.tika.grpc.v2alpha1.ParsedDocument; +import org.apache.tika.grpc.v2alpha1.RequestError; +import org.apache.tika.grpc.v2alpha1.TikaServiceGrpc; +import org.apache.tika.grpc.v2alpha1.ValueType; +import org.apache.tika.grpc.v2alpha1.Values; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.schema.MetadataKeyValidator; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.pipes.api.PipesResult; +import org.apache.tika.pipes.api.fetcher.FetcherNotFoundException; +import org.apache.tika.pipes.core.PipesException; + +/** + * The experimental v2alpha1 service: what /rmeta returns, as protobuf. Goes through the same + * {@link FetchParseRuntime} as v1, so it reaches nothing v1 does not. + */ +class TikaV2Alpha1ServerImpl extends TikaServiceGrpc.TikaServiceImplBase { + + private static final Logger LOG = LoggerFactory.getLogger(TikaV2Alpha1ServerImpl.class); + private static final String CONTENT_KEY = TikaCoreProperties.TIKA_CONTENT.getName(); + private static final int MAX_ECHOED_ID = 200; + + private final FetchParseRuntime runtime; + private final MetadataKeyValidator registry; + + TikaV2Alpha1ServerImpl(FetchParseRuntime runtime) { + this.runtime = runtime; + this.registry = MetadataKeyValidator.fromClasspath(); + } + + @Override + public void fetchAndParse(FetchAndParseRequest request, + StreamObserver responseObserver) { + try { + responseObserver.onNext(run(request.getFetcherId(), request.getFetchKey(), + request.getParseContextJson(), request.getAdditionalFetchConfigJson())); + responseObserver.onCompleted(); + } catch (StatusException e) { + responseObserver.onError(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + responseObserver.onError(interrupted()); + } + } + + @Override + public void fetchAndParseStream(FetchAndParseStreamRequest request, + StreamObserver responseObserver) { + try { + FetchAndParseResponse response = run(request.getFetcherId(), request.getFetchKey(), + request.getParseContextJson(), request.getAdditionalFetchConfigJson()); + for (ParsedDocument document : response.getDocumentsList()) { + responseObserver.onNext( + FetchAndParseStreamResponse.newBuilder().setDocument(document).build()); + } + responseObserver.onCompleted(); + } catch (StatusException e) { + responseObserver.onError(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + responseObserver.onError(interrupted()); + } + } + + @Override + public StreamObserver fetchAndParseBidi( + StreamObserver responseObserver) { + return new StreamObserver<>() { + // set once the stream is closed with an error; later requests have nowhere to go + private boolean closed; + + @Override + public void onNext(FetchAndParseBidiRequest request) { + if (closed) { + return; + } + FetchAndParseBidiResponse.Builder response = + FetchAndParseBidiResponse.newBuilder().setRequestId(request.getRequestId()); + try { + response.setResult(run(request.getFetcherId(), request.getFetchKey(), + request.getParseContextJson(), request.getAdditionalFetchConfigJson())); + } catch (StatusException e) { + response.setError(toRequestError(e.getStatus())); + } catch (InterruptedException e) { + // shutdown: the only failure that ends the stream + Thread.currentThread().interrupt(); + closed = true; + responseObserver.onError(interrupted()); + return; + } + responseObserver.onNext(response.build()); + } + + @Override + public void onError(Throwable throwable) { + LOG.debug("v2alpha1 bidi stream ended by the client", throwable); + } + + @Override + public void onCompleted() { + if (!closed) { + responseObserver.onCompleted(); + } + } + }; + } + + /** Failures that are not about the document are thrown as a gRPC status. */ + private FetchAndParseResponse run(String fetcherId, String fetchKey, String parseContextJson, + String additionalFetchConfigJson) + throws StatusException, InterruptedException { + Status denial = runtime.perRequestConfigDenial(additionalFetchConfigJson, parseContextJson); + if (denial != null) { + throw denial.asException(); + } + ParseContext parseContext = + runtime.requestParseContext(fetcherId, parseContextJson, additionalFetchConfigJson); + String resolvedFetcherId; + try { + resolvedFetcherId = runtime.resolveFetcherId(fetcherId); + } catch (FetcherNotFoundException e) { + throw Status.NOT_FOUND.withDescription("no fetcher named '" + echo(fetcherId) + "'") + .asException(); + } catch (TikaException | IOException | RuntimeException e) { + LOG.warn("could not load fetcher '{}'", fetcherId, e); + throw Status.UNAVAILABLE.withDescription("fetcher '" + echo(fetcherId) + + "' could not be loaded").asException(); + } + PipesResult result; + try { + result = runtime.fetchAndParse(resolvedFetcherId, fetchKey, parseContext); + } catch (IOException | PipesException e) { + LOG.warn("fetch and parse failed", e); + throw Status.INTERNAL.withDescription("fetch and parse failed").asException(); + } + Status failure = serviceFailure(result.status(), fetcherId); + if (failure != null) { + throw failure.asException(); + } + return toResponse(fetchKey, result); + } + + /** The gRPC status for a pipes result that is not about the document, else {@code null}. */ + static Status serviceFailure(PipesResult.RESULT_STATUS status, String fetcherId) { + // exhaustive on purpose: a new RESULT_STATUS fails compilation until it is classified here + return switch (status) { + case CLIENT_UNAVAILABLE_WITHIN_MS -> Status.RESOURCE_EXHAUSTED + .withDescription("no pipes worker became free in time; retry later"); + case FAILED_TO_INITIALIZE -> Status.UNAVAILABLE + .withDescription("the pipes worker failed to start"); + case FETCHER_INITIALIZATION_EXCEPTION -> Status.UNAVAILABLE.withDescription( + "fetcher '" + echo(fetcherId) + "' failed to initialize in the pipes worker"); + case EMITTER_INITIALIZATION_EXCEPTION -> Status.UNAVAILABLE + .withDescription("an emitter failed to initialize in the pipes worker"); + case FETCHER_NOT_FOUND -> Status.NOT_FOUND.withDescription( + "the pipes worker has no fetcher '" + echo(fetcherId) + "'"); + case EMITTER_NOT_FOUND -> Status.NOT_FOUND + .withDescription("the pipes worker has no such emitter"); + case PRESET_NOT_FOUND -> Status.NOT_FOUND.withDescription("unknown preset"); + case FETCH_EXCEPTION, EMIT_EXCEPTION, PAYLOAD_LIMIT_EXCEEDED, OOM, TIMEOUT, + UNSPECIFIED_CRASH, EMPTY_OUTPUT, PARSE_SUCCESS, PARSE_SUCCESS_WITH_EXCEPTION, + PARSE_EXCEPTION_NO_EMIT, EMIT_SUCCESS, EMIT_SUCCESS_PARSE_EXCEPTION, + EMIT_SUCCESS_PASSBACK, PARTIAL_TIMEOUT -> null; + }; + } + + private FetchAndParseResponse toResponse(String fetchKey, PipesResult result) { + FetchAndParseResponse.Builder response = FetchAndParseResponse.newBuilder() + .setFetchKey(fetchKey) + .setPipesStatus(result.status().name()); + if (StringUtils.isNotBlank(result.message())) { + response.setMessage(result.message()); + } + if (result.emitData() != null && result.emitData().getMetadataList() != null) { + for (Metadata metadata : result.emitData().getMetadataList()) { + response.addDocuments(toDocument(metadata)); + } + } + return response.build(); + } + + ParsedDocument toDocument(Metadata metadata) { + ParsedDocument.Builder document = ParsedDocument.newBuilder(); + String path = metadata.get(TikaCoreProperties.EMBEDDED_ID_PATH); + if (path != null) { + document.setEmbeddedPath(path); + } + String content = metadata.get(CONTENT_KEY); + if (content != null) { + document.setContent(content); + } + for (String name : metadata.names()) { + if (name.equals(CONTENT_KEY)) { + continue; + } + document.putMetadata(name, Values.newBuilder() + .addAllValues(Arrays.asList(metadata.getValues(name))) + .setType(valueType(registry.valueTypeOf(name))) + .setCardinality(cardinality(registry.cardinalityOf(name))) + .build()); + } + return document.build(); + } + + // only the types that change how a value parses; everything else reads as text + static ValueType valueType(String registered) { + if (registered == null) { + return ValueType.VALUE_TYPE_UNSPECIFIED; + } + return switch (registered) { + case "BOOLEAN" -> ValueType.VALUE_TYPE_BOOLEAN; + case "INTEGER" -> ValueType.VALUE_TYPE_INTEGER; + case "REAL" -> ValueType.VALUE_TYPE_REAL; + case "RATIONAL" -> ValueType.VALUE_TYPE_RATIONAL; + case "DATE" -> ValueType.VALUE_TYPE_DATE; + case "URI" -> ValueType.VALUE_TYPE_URI; + case "JSON" -> ValueType.VALUE_TYPE_JSON; + default -> ValueType.VALUE_TYPE_UNSPECIFIED; + }; + } + + static Cardinality cardinality(String registered) { + if (registered == null) { + return Cardinality.CARDINALITY_UNSPECIFIED; + } + return switch (registered) { + case "BAG" -> Cardinality.CARDINALITY_BAG; + case "SEQ" -> Cardinality.CARDINALITY_SEQ; + case "ALT" -> Cardinality.CARDINALITY_ALT; + default -> Cardinality.CARDINALITY_UNSPECIFIED; + }; + } + + private static RequestError toRequestError(Status status) { + return RequestError.newBuilder() + .setCode(status.getCode().value()) + .setMessage(StringUtils.defaultString(status.getDescription())) + .build(); + } + + private static StatusException interrupted() { + return Status.UNAVAILABLE.withDescription("the server is shutting down").asException(); + } + + private static String echo(String id) { + return StringUtils.abbreviate(StringUtils.defaultString(id), MAX_ECHOED_ID); + } +} diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImplTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImplTest.java new file mode 100644 index 00000000000..7a1d4f9bea7 --- /dev/null +++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaV2Alpha1ServerImplTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.grpc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import com.asarkar.grpc.test.GrpcCleanupExtension; +import com.asarkar.grpc.test.Resources; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.grpc.Grpc; +import io.grpc.InsecureChannelCredentials; +import io.grpc.InsecureServerCredentials; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.stub.StreamObserver; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.apache.tika.grpc.v2alpha1.Cardinality; +import org.apache.tika.grpc.v2alpha1.FetchAndParseBidiRequest; +import org.apache.tika.grpc.v2alpha1.FetchAndParseBidiResponse; +import org.apache.tika.grpc.v2alpha1.FetchAndParseRequest; +import org.apache.tika.grpc.v2alpha1.FetchAndParseResponse; +import org.apache.tika.grpc.v2alpha1.FetchAndParseStreamRequest; +import org.apache.tika.grpc.v2alpha1.ParsedDocument; +import org.apache.tika.grpc.v2alpha1.TikaServiceGrpc; +import org.apache.tika.grpc.v2alpha1.ValueType; +import org.apache.tika.grpc.v2alpha1.Values; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.serialization.config.JsonConfigHelper; + +@ExtendWith(GrpcCleanupExtension.class) +public class TikaV2Alpha1ServerImplTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + // a file-system fetcher rooted at target/, from tika-pipes-test-config.json + private static final String FETCHER_ID = "nick1.is.cool.super-fs"; + private static final String DOC_DIR = "v2alpha1-" + UUID.randomUUID(); + private static final String ZIP_KEY = DOC_DIR + "/two-texts.zip"; + private static final String BIG_KEY = DOC_DIR + "/big.txt"; + + static Path tikaConfig = Paths.get("target", "tika-config-v2alpha1-" + UUID.randomUUID() + ".json"); + static Path tikaConfigUnlocked = + Paths.get("target", "tika-config-v2alpha1-unlocked-" + UUID.randomUUID() + ".json"); + + private final List services = new ArrayList<>(); + + @BeforeAll + static void init() throws Exception { + Path target = Paths.get("target").toAbsolutePath(); + Map replacements = new HashMap<>(); + replacements.put("JAVA_PATH", Paths.get(System.getProperty("java.home"), "bin", "java")); + replacements.put("FETCHER_BASE_PATH", target); + replacements.put("PLUGIN_ROOTS", target.resolve("plugins")); + JsonConfigHelper.writeConfigFromResource("/tika-pipes-test-config.json", + TikaV2Alpha1ServerImplTest.class, replacements, tikaConfig); + + ObjectNode root = (ObjectNode) OBJECT_MAPPER.readTree(tikaConfig.toFile()); + root.set("grpc", OBJECT_MAPPER.createObjectNode().put("allowPerRequestConfig", true)); + Files.writeString(tikaConfigUnlocked, OBJECT_MAPPER.writeValueAsString(root), + StandardCharsets.UTF_8); + + Path docs = Files.createDirectories(target.resolve(DOC_DIR)); + try (OutputStream os = Files.newOutputStream(docs.resolve("two-texts.zip")); + ZipOutputStream zip = new ZipOutputStream(os)) { + for (String name : List.of("alpha", "beta")) { + ZipEntry entry = new ZipEntry(name + ".txt"); + entry.setTime(1_600_000_000_000L); + zip.putNextEntry(entry); + zip.write((name + "-marker text").getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + } + // ~5 MB of text: over gRPC's default 4 MB client limit once extracted + String line = "the quick brown fox jumps over the lazy dog 0123456789\n"; + Files.writeString(docs.resolve("big.txt"), line.repeat(5 * 1024 * 1024 / line.length() + 1), + StandardCharsets.UTF_8); + } + + @AfterAll + static void clean() throws Exception { + Files.deleteIfExists(tikaConfig); + Files.deleteIfExists(tikaConfigUnlocked); + FileUtils.deleteDirectory(Paths.get("target").resolve(DOC_DIR).toFile()); + } + + @AfterEach + void closeServices() throws Exception { + for (TikaGrpcServerImpl service : services) { + service.shutdown(); + service.postShutdown(); + } + services.clear(); + } + + private ManagedChannel start(Resources resources, Path config) throws Exception { + String name = InProcessServerBuilder.generateName(); + Server server = InProcessServerBuilder.forName(name).directExecutor() + .addService(newService(config)).build().start(); + resources.register(server, Duration.ofSeconds(10)); + ManagedChannel channel = InProcessChannelBuilder.forName(name).directExecutor().build(); + resources.register(channel, Duration.ofSeconds(10)); + return channel; + } + + // in-process transport hands messages over as objects and never checks their size + private ManagedChannel startOverNetwork(Resources resources, Path config) throws Exception { + Server server = Grpc.newServerBuilderForPort(0, InsecureServerCredentials.create()) + .addService(newService(config)).build().start(); + resources.register(server, Duration.ofSeconds(10)); + ManagedChannel channel = Grpc.newChannelBuilderForAddress("localhost", server.getPort(), + InsecureChannelCredentials.create()).build(); + resources.register(channel, Duration.ofSeconds(10)); + return channel; + } + + private TikaV2Alpha1ServerImpl newService(Path config) throws Exception { + TikaGrpcServerImpl v1 = new TikaGrpcServerImpl(config.toAbsolutePath().toString()); + services.add(v1); + return new TikaV2Alpha1ServerImpl(v1.runtime); + } + + private static FetchAndParseRequest request(String fetcherId, String fetchKey) { + return FetchAndParseRequest.newBuilder().setFetcherId(fetcherId).setFetchKey(fetchKey).build(); + } + + @Test + public void unaryReturnsTheContainerAndEachEmbeddedDocument(Resources resources) throws Exception { + FetchAndParseResponse response = TikaServiceGrpc.newBlockingStub(start(resources, tikaConfig)) + .fetchAndParse(request(FETCHER_ID, ZIP_KEY)); + + assertEquals("PARSE_SUCCESS", response.getPipesStatus(), response.getMessage()); + assertEquals(3, response.getDocumentsCount()); + ParsedDocument container = response.getDocuments(0); + assertEquals("", container.getEmbeddedPath()); + for (int i = 1; i < 3; i++) { + ParsedDocument embedded = response.getDocuments(i); + assertTrue(embedded.getEmbeddedPath().startsWith("/"), embedded.getEmbeddedPath()); + assertFalse(embedded.containsMetadata("tk:content"), "content is not repeated in metadata"); + } + assertTrue(response.getDocuments(1).getContent().contains("alpha-marker")); + assertTrue(response.getDocuments(2).getContent().contains("beta-marker")); + + // hints from the registry: INTEGER, DATE, a BAG of text, and an unhinted key + ParsedDocument alpha = response.getDocuments(1); + assertEquals(ValueType.VALUE_TYPE_INTEGER, alpha.getMetadataOrThrow("tk:embedded-depth").getType()); + assertEquals(ValueType.VALUE_TYPE_DATE, alpha.getMetadataOrThrow("fs:modified").getType()); + Values parsedBy = alpha.getMetadataOrThrow("tk:parsed-by"); + assertEquals(ValueType.VALUE_TYPE_UNSPECIFIED, parsedBy.getType()); + assertEquals(Cardinality.CARDINALITY_BAG, parsedBy.getCardinality()); + assertTrue(parsedBy.getValuesCount() >= 1); + } + + @Test + public void streamSendsTheSameDocumentsOnePerMessage(Resources resources) throws Exception { + ManagedChannel channel = start(resources, tikaConfig); + List unary = TikaServiceGrpc.newBlockingStub(channel) + .fetchAndParse(request(FETCHER_ID, ZIP_KEY)).getDocumentsList(); + List streamed = new ArrayList<>(); + TikaServiceGrpc.newBlockingStub(channel).fetchAndParseStream(FetchAndParseStreamRequest + .newBuilder().setFetcherId(FETCHER_ID).setFetchKey(ZIP_KEY).build()) + .forEachRemaining(r -> streamed.add(r.getDocument())); + + assertEquals(unary.size(), streamed.size()); + for (int i = 0; i < unary.size(); i++) { + assertEquals(unary.get(i).getEmbeddedPath(), streamed.get(i).getEmbeddedPath()); + assertEquals(unary.get(i).getContent(), streamed.get(i).getContent()); + } + } + + @Test + public void anUnknownFetcherIsNotFound(Resources resources) throws Exception { + StatusRuntimeException e = assertThrows( + StatusRuntimeException.class, () -> TikaServiceGrpc + .newBlockingStub(start(resources, tikaConfig)) + .fetchAndParse(request("no-such-fetcher", ZIP_KEY))); + assertEquals(Status.Code.NOT_FOUND, e.getStatus().getCode()); + } + + @Test + public void aMissingFileIsADocumentOutcomeWithAMessage(Resources resources) throws Exception { + FetchAndParseResponse response = TikaServiceGrpc.newBlockingStub(start(resources, tikaConfig)) + .fetchAndParse(request(FETCHER_ID, DOC_DIR + "/does-not-exist.txt")); + assertEquals("FETCH_EXCEPTION", response.getPipesStatus()); + assertFalse(response.getMessage().isEmpty()); + } + + @Test + public void perRequestConfigIsDeniedUnlessAllowed(Resources resources) throws Exception { + FetchAndParseRequest withConfig = request(FETCHER_ID, ZIP_KEY).toBuilder() + .setParseContextJson("{}").build(); + StatusRuntimeException e = assertThrows( + StatusRuntimeException.class, () -> TikaServiceGrpc + .newBlockingStub(start(resources, tikaConfig)).fetchAndParse(withConfig)); + assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); + } + + @Test + public void bidiAnswersEveryRequestAndReportsFailuresInBand(Resources resources) throws Exception { + ManagedChannel channel = start(resources, tikaConfig); + Recording responses = new Recording(); + StreamObserver requests = + TikaServiceGrpc.newStub(channel).fetchAndParseBidi(responses); + requests.onNext(bidi("1", FETCHER_ID, ZIP_KEY, "")); + requests.onNext(bidi("2", "no-such-fetcher", ZIP_KEY, "")); + // denied per request on a locked server, not for the whole stream + requests.onNext(bidi("3", FETCHER_ID, ZIP_KEY, "{}")); + requests.onNext(bidi("4", FETCHER_ID, ZIP_KEY, "")); + requests.onCompleted(); + assertTrue(responses.done.await(60, TimeUnit.SECONDS)); + + assertTrue(responses.errors.isEmpty(), String.valueOf(responses.errors)); + assertEquals(4, responses.responses.size()); + Map byId = new HashMap<>(); + responses.responses.forEach(r -> byId.put(r.getRequestId(), r)); + assertEquals(3, byId.get("1").getResult().getDocumentsCount()); + assertEquals(Status.Code.NOT_FOUND.value(), byId.get("2").getError().getCode()); + assertEquals(Status.Code.PERMISSION_DENIED.value(), byId.get("3").getError().getCode()); + assertEquals(3, byId.get("4").getResult().getDocumentsCount()); + } + + @Test + public void aReplyOverFourMegabytesNeedsARaisedClientLimit(Resources resources) throws Exception { + ManagedChannel channel = startOverNetwork(resources, tikaConfig); + StatusRuntimeException e = assertThrows( + StatusRuntimeException.class, () -> TikaServiceGrpc.newBlockingStub(channel) + .fetchAndParse(request(FETCHER_ID, BIG_KEY))); + assertEquals(Status.Code.RESOURCE_EXHAUSTED, e.getStatus().getCode()); + + FetchAndParseResponse response = TikaServiceGrpc.newBlockingStub(channel) + .withMaxInboundMessageSize(64 * 1024 * 1024) + .fetchAndParse(request(FETCHER_ID, BIG_KEY)); + assertTrue(response.getDocuments(0).getContent().length() > 4 * 1024 * 1024); + } + + @Test + public void hintsComeFromTheRegistry() { + Metadata metadata = new Metadata(); + metadata.add("pdf:chars-per-page", "10"); + metadata.add("pdf:chars-per-page", "20"); + metadata.set("pdf:encrypted", "false"); + metadata.set("x-not-registered", "anything"); + ParsedDocument document = new TikaV2Alpha1ServerImpl(null).toDocument(metadata); + + Values chars = document.getMetadataOrThrow("pdf:chars-per-page"); + assertEquals(List.of("10", "20"), chars.getValuesList()); + assertEquals(ValueType.VALUE_TYPE_INTEGER, chars.getType()); + assertEquals(Cardinality.CARDINALITY_SEQ, chars.getCardinality()); + assertEquals(ValueType.VALUE_TYPE_BOOLEAN, document.getMetadataOrThrow("pdf:encrypted").getType()); + Values unknown = document.getMetadataOrThrow("x-not-registered"); + assertEquals(ValueType.VALUE_TYPE_UNSPECIFIED, unknown.getType()); + assertEquals(Cardinality.CARDINALITY_UNSPECIFIED, unknown.getCardinality()); + } + + private static FetchAndParseBidiRequest bidi(String id, String fetcherId, String fetchKey, + String parseContextJson) { + return FetchAndParseBidiRequest.newBuilder().setRequestId(id).setFetcherId(fetcherId) + .setFetchKey(fetchKey).setParseContextJson(parseContextJson).build(); + } + + private static final class Recording implements StreamObserver { + private final List responses = + Collections.synchronizedList(new ArrayList<>()); + private final List errors = Collections.synchronizedList(new ArrayList<>()); + private final CountDownLatch done = new CountDownLatch(1); + + @Override + public void onNext(FetchAndParseBidiResponse response) { + responses.add(response); + } + + @Override + public void onError(Throwable t) { + errors.add(t); + done.countDown(); + } + + @Override + public void onCompleted() { + done.countDown(); + } + } +} diff --git a/tika-metadata-schema/README.md b/tika-metadata-schema/README.md index 3360088d52f..df6503acca4 100644 --- a/tika-metadata-schema/README.md +++ b/tika-metadata-schema/README.md @@ -45,6 +45,17 @@ Installs the dependency modules, regenerates the registries via the forked-exec the key-count diff, and runs the gate tests — see `.skills/devs/metadata-schema/SKILL.md` for flags and the manual steps this replaces. +**Type changes are declared, never silent.** A key's `valueType` or `cardinality` is part of what +clients rely on: gRPC v2 type hints and typed `/rmeta` JSON both come from it. `MetadataTypeChangesTest` +compares the registry with a committed baseline +(`src/test/resources/org/apache/tika/metadata/schema/metadata-key-types-baseline.json`). Any change +must be listed in `type-changes.json` next to it, as +`{"key": "...", "from": "INTEGER/SIMPLE", "to": "TEXT/SIMPLE", "jira": "TIKA-NNNN"}`, with a CHANGES +row. At a release, refresh the baseline and clear the list: +``` +./mvnw -pl tika-metadata-schema test -Dtest=MetadataTypeChangesTest -Dtika.metadata.refreshTypeBaseline=true +``` + ## `metadata-open-namespaces.json` — the open sets (generated + gated) The **prefixes** under which parsers mint file-controlled key names at runtime — names that are not `Property` constants, so the individual keys cannot be enumerated (scraped HTML `` under diff --git a/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/MetadataKeyValidator.java b/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/MetadataKeyValidator.java index e57f22003bb..8b9268be969 100644 --- a/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/MetadataKeyValidator.java +++ b/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/MetadataKeyValidator.java @@ -56,6 +56,8 @@ public enum Classification { CLOSED, OPEN, TEMPLATE, UNKNOWN } private final List openPrefixes; // longest-first: most specific prefix wins private final Map keyModules; // closed key -> supplying module private final Map prefixModules; // open prefix -> supplying module + private final Map keyValueTypes; // closed key -> Property.ValueType name + private final Map keyCardinalities; // closed key -> Property.PropertyType name MetadataKeyValidator(Set closedKeys, List openPrefixes) { this(closedKeys, openPrefixes, Map.of(), Map.of()); @@ -66,6 +68,14 @@ public enum Classification { CLOSED, OPEN, TEMPLATE, UNKNOWN } * registries) or unattributed for a given entry. */ MetadataKeyValidator(Set closedKeys, List openPrefixes, Map keyModules, Map prefixModules) { + this(closedKeys, openPrefixes, keyModules, prefixModules, Map.of(), Map.of()); + } + + MetadataKeyValidator(Set closedKeys, List openPrefixes, + Map keyModules, Map prefixModules, + Map keyValueTypes, Map keyCardinalities) { + this.keyValueTypes = Map.copyOf(keyValueTypes); + this.keyCardinalities = Map.copyOf(keyCardinalities); this.closedKeys = Set.copyOf(closedKeys); List sorted = new ArrayList<>(openPrefixes); sorted.sort(Comparator.comparingInt(String::length).reversed()); @@ -78,7 +88,19 @@ public enum Classification { CLOSED, OPEN, TEMPLATE, UNKNOWN } public static MetadataKeyValidator fromClasspath() { return new MetadataKeyValidator( new HashSet<>(readValues(KEYS, "key")), readValues(OPEN, "prefix"), - readFieldMap(KEYS, "key", "module"), readFieldMap(OPEN, "prefix", "module")); + readFieldMap(KEYS, "key", "module"), readFieldMap(OPEN, "prefix", "module"), + readFieldMap(KEYS, "key", "valueType"), readFieldMap(KEYS, "key", "cardinality")); + } + + /** The registered {@code Property.ValueType} name of a closed key, or {@code null}. */ + public String valueTypeOf(String key) { + return keyValueTypes.get(key); + } + + /** The registered {@code Property.PropertyType} name (SIMPLE, BAG, SEQ, ...) of a closed key, + * or {@code null}. */ + public String cardinalityOf(String key) { + return keyCardinalities.get(key); } public Classification classify(String key) { diff --git a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataTypeChangesTest.java b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataTypeChangesTest.java new file mode 100644 index 00000000000..5f0753ee1d6 --- /dev/null +++ b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataTypeChangesTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.metadata.schema; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.junit.jupiter.api.Test; + +/** + * A key's valueType or cardinality may change only when type-changes.json declares it. Clients + * that read typed values (gRPC type hints, typed /rmeta JSON) see such a change, so it must be + * reviewed and noted in CHANGES, never slip in through a regenerated registry. + *

+ * At a release, refresh the baseline and clear type-changes.json: + * {@code ./mvnw -pl tika-metadata-schema test -Dtest=MetadataTypeChangesTest + * -Dtika.metadata.refreshTypeBaseline=true} + */ +public class MetadataTypeChangesTest { + + private static final String REGISTRY = "/org/apache/tika/metadata/metadata-keys.json"; + private static final Path DIR = Paths.get("src/test/resources/org/apache/tika/metadata/schema"); + private static final Path BASELINE = DIR.resolve("metadata-key-types-baseline.json"); + private static final Path CHANGES = DIR.resolve("type-changes.json"); + private static final Pattern OBJECT = Pattern.compile("\\{[^{}]*}"); + + @Test + public void typeChangesAreDeclared() throws Exception { + Map current = types(readRegistry()); + if (Boolean.getBoolean("tika.metadata.refreshTypeBaseline")) { + writeBaseline(current); + Files.writeString(CHANGES, "[\n]\n", StandardCharsets.UTF_8); + return; + } + Map baseline = types(Files.readString(BASELINE, StandardCharsets.UTF_8)); + + Map declared = new TreeMap<>(); + Matcher m = OBJECT.matcher(Files.readString(CHANGES, StandardCharsets.UTF_8)); + while (m.find()) { + String record = m.group(); + String key = field(record, "key"); + assertNotNull(key, "type-changes.json record without a key: " + record); + String jira = field(record, "jira"); + assertTrue(jira != null && !jira.isBlank(), "type-changes.json: " + key + " needs a jira"); + declared.put(key, field(record, "from") + " -> " + field(record, "to")); + } + + List undeclared = new ArrayList<>(); + for (Map.Entry e : baseline.entrySet()) { + String now = current.get(e.getKey()); + if (now == null || now.equals(e.getValue())) { + continue; + } + String change = e.getValue() + " -> " + now; + if (!change.equals(declared.remove(e.getKey()))) { + undeclared.add(e.getKey() + ": " + change); + } + } + assertTrue(undeclared.isEmpty(), "metadata key types changed without a type-changes.json entry " + + "(add {\"key\", \"from\", \"to\", \"jira\"} and a CHANGES row, or revert): " + + undeclared); + assertTrue(declared.isEmpty(), + "type-changes.json declares changes the registry does not show: " + declared); + } + + /** key -> "VALUETYPE/CARDINALITY" */ + private static Map types(String json) { + Map types = new TreeMap<>(); + Matcher m = OBJECT.matcher(json); + while (m.find()) { + String record = m.group(); + String key = field(record, "key"); + if (key != null) { + types.put(key, field(record, "valueType") + "/" + field(record, "cardinality")); + } + } + return types; + } + + private static String field(String record, String name) { + Matcher m = Pattern.compile("\"" + Pattern.quote(name) + "\"\\s*:\\s*\"((?:[^\"\\\\]|\\\\.)*)\"") + .matcher(record); + return m.find() ? m.group(1).replace("\\\"", "\"").replace("\\\\", "\\") : null; + } + + private static String readRegistry() throws Exception { + try (InputStream in = MetadataTypeChangesTest.class.getResourceAsStream(REGISTRY)) { + assertNotNull(in, "missing " + REGISTRY); + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static void writeBaseline(Map types) throws Exception { + StringBuilder sb = new StringBuilder("[\n"); + int i = 0; + for (Map.Entry e : types.entrySet()) { + String[] parts = e.getValue().split("/", 2); + sb.append(" {\"key\": \"").append(e.getKey().replace("\\", "\\\\").replace("\"", "\\\"")) + .append("\", \"valueType\": \"").append(parts[0]) + .append("\", \"cardinality\": \"").append(parts[1]).append("\"}") + .append(++i < types.size() ? ",\n" : "\n"); + } + sb.append("]\n"); + Files.createDirectories(DIR); + Files.writeString(BASELINE, sb.toString(), StandardCharsets.UTF_8); + } +} diff --git a/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/metadata-key-types-baseline.json b/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/metadata-key-types-baseline.json new file mode 100644 index 00000000000..fc48f558910 --- /dev/null +++ b/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/metadata-key-types-baseline.json @@ -0,0 +1,805 @@ +[ + {"key": "Camera:MicroVideo", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Camera:MicroVideoOffset", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Camera:MicroVideoPresentationTimestampUs", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Camera:MicroVideoVersion", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Camera:MotionPhoto", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Camera:MotionPhotoPresentationTimestampUs", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Camera:MotionPhotoVersion", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Content-Disposition", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Content-Encoding", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Content-Language", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Content-Length", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Content-Location", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Content-MD5", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Content-Type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:CiAdrCity", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:CiAdrCtry", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:CiAdrExtadr", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpCore:CiAdrPcode", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:CiAdrRegion", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:CiEmailWork", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpCore:CiTelWork", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpCore:CiUrlWork", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpCore:CountryCode", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:CreatorContactInfo", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:IntellectualGenre", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:Location", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpCore:Scene", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpCore:SubjectCode", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:AOCopyrightNotice", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:AOCreator", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:AODateCreated", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:AOSource", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:AOSourceInvNo", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:AOTitle", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:AddlModelInfo", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:ArtworkOrObject", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:CVterm", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:DigImageGUID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:DigitalSourceType", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:Event", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:IptcLastEdited", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:LocationCreated", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:LocationCreatedCity", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:LocationCreatedCountryCode", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:LocationCreatedCountryName", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:LocationCreatedProvinceState", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:LocationCreatedSublocation", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:LocationCreatedWorldRegion", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:LocationShown", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:LocationShownCity", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:LocationShownCountryCode", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:LocationShownCountryName", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:LocationShownProvinceState", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:LocationShownSublocation", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:LocationShownWorldRegion", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:MaxAvailHeight", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:MaxAvailWidth", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "Iptc4xmpExt:ModelAge", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:OrganisationInImageCode", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:OrganisationInImageName", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:PersonInImage", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:RegItemId", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:RegOrgId", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Iptc4xmpExt:RegistryId", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "Location", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "access-permission:assemble-document", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "access-permission:can-modify", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "access-permission:can-print", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "access-permission:can-print-faithful", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "access-permission:extract-content", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "access-permission:extract-for-accessibility", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "access-permission:fill-in-form", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "access-permission:modify-annotations", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:bitrate", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "audio:bits-per-sample", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "audio:channels", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "audio:disc-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "audio:encoding", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:fourcc", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:has-drm", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "audio:is-variable-bitrate", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "audio:raw-disc-number", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:raw-track-number", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:spi-author", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:spi-comment", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:spi-copyright", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:spi-date", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:spi-duration", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:spi-quality", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "audio:spi-title", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "audio:track-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "aux:Lens", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "aux:LensID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "aux:LensInfo", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "aux:SerialNumber", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:Conventions", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:acknowledgement", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:cmd_ln", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:comment", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:contact", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:experiment_id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:history", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "cf:institution", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:model_name_english", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:prg_ID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:project_id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:realization", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:references", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:source", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cf:table_id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cp:category", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cp:contentStatus", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cp:lastModifiedBy", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cp:lastPrinted", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "cp:revision", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "cp:version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "csv:delimiter", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "csv:num-columns", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "csv:num-rows", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "database:column-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "database:column-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "database:row-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "database:table-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:contributor", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:coverage", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:creator", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:date", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "dc:description", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:format", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "dc:identifier", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:language", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:publisher", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:relation", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:rights", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:source", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:subject", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:title", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dc:type", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "dcterms:created", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "dcterms:modified", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "dwg:application-comment", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "dwg:application-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "dwg:application-version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "dwg:product-info", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "emf:icon-only", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "emf:icon-string", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "epub:rendition:layout", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "epub:version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "exif:DateTimeOriginal", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "exif:ExposureTime", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "exif:FNumber", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "exif:Flash", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "exif:FocalLength", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "exif:IsoSpeedRatings", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "exif:PageCount", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "extended-properties:AppVersion", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "extended-properties:Application", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "extended-properties:Company", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "extended-properties:DocSecurity", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "extended-properties:HiddenSlides", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "extended-properties:Manager", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "extended-properties:Notes", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "extended-properties:PresentationFormat", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "extended-properties:Template", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "extended-properties:TotalTime", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "extended-properties:doc-security-string", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "external-process:exit-value", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "external-process:stderr", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "external-process:stderr-length", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "external-process:stderr-truncated", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "external-process:stdout", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "external-process:stdout-length", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "external-process:stdout-truncated", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "external-process:timeout", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "file:mime", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "flv:hasAudio", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "flv:hasVideo", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "font:name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "fs:accessed", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "fs:created", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "fs:modified", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "gdal:coordinate-system", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "gdal:driver", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "gdal:files", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "gdal:lower-left", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "gdal:lower-right", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "gdal:size", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "gdal:upper-left", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "gdal:upper-right", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geo:alt", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "geo:lat", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "geo:long", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "geo:timestamp", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "geogebra:app-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geogebra:app-version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geogebra:date", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geogebra:format-version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geogebra:id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geogebra:toolName", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "geotopic:latitude", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geotopic:longitude", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "geotopic:name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:abstract", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:address", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:affiliation", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:authors", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:class", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:error", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:full-affiliations", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:keyword", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "grobid:tei:language", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:title", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "grobid:tei:xml-source", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "hdf:file-type-description", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "html:scriptSrc", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "icon:hotspot-x", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "icon:hotspot-y", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "icon:image-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "icon:images", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "idml:master-spread-page-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "idml:spread-page-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "iso19115:access-constraints", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:character-set", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:citation-date", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:cited-responsible-party-email", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:cited-responsible-party-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:cited-responsible-party-organization-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:cited-responsible-party-position-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:cited-responsible-party-role", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:contact-party-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:contact-role", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:date-info", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:distribution-format-specification-alternative-title", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:distributor-contact", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:distributor-organization-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:geographic-identifier-authority-alternative-title", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:geographic-identifier-authority-date", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:geographic-identifier-authority-title", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:geographic-identifier-code", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:identification-info-status", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:identification-info-topic-category", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:metadata-identifier-code", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "iso19115:metadata-resource-scope", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:metadata-standard-edition", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:metadata-standard-title", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:other-constraints", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:parent-metadata-title", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "iso19115:resource-format-specification-alternative-title", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:thesaurus-name-date", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:transfer-options-online-description", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:transfer-options-online-function", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:transfer-options-online-linkage", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:transfer-options-online-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:transfer-options-online-profile", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:transfer-options-online-protocol", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iso19115:use-constraints", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iwork:build-version-history", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "iwork:document-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "keynote:slides-height", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "keynote:slides-width", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "machine:architectureBits", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "machine:endian", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "machine:machineType", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "machine:platform", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:content-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:content-location", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:display-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:extension", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:file-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:flags", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:hidden", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:language", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:long-file-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:long-path-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:attach:mime", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:body-types-processed", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "mapi:conversation-index", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:conversation-topic", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:creation-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:deferred-delivery-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:deliver-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:expiry-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:from-representing-email", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:from-representing-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:importance", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "mapi:in-reply-to-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:internet-message-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:internet-references", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "mapi:is-flagged", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "mapi:last-modification-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:latest-delivery-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:message-class", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:message-class-raw", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:message-delivery-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:message-download-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:msg-client-submit-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:msg-submission-accepted-at-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:msg-submission-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:original-delivery-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:original-submit-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:priority", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "mapi:provider-submit-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:receipt-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:recipients-string", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mapi:reply-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:report-time", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "mapi:sent-by-server-type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "mdb:password", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "message:bcc", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:bcc-display-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:bcc-email", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:bcc-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:cc", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:cc-display-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:cc-email", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:cc-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:from", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:from-email", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:from-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:recipient-address", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:to", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:to-display-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:to-email", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "message:to-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "meta:author", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "meta:character-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:character-count-with-spaces", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:creation-date", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "meta:image-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:initial-author", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "meta:keyword", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "meta:last-author", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "meta:line-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:object-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:page-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:paragraph-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:print-date", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "meta:save-date", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "meta:slide-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:table-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "meta:word-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "midi:division-type", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "midi:patches", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "midi:tracks", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "mif:master-page-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "mif:reference-page-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "mif:total-page-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "mp3:version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "msoffice:comment-person-display-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "msoffice:doc:has-attached-template", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:doc:has-framesets", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:doc:has-hidden-text", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:doc:has-mail-merge", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:doc:has-subdocuments", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:embedded-storage-class-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-data-connections", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-dde-links", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-external-links", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-external-pivot-data", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-hidden-cols", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-hidden-rows", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-hidden-sheets", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-power-query", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-very-hidden-sheets", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:has-web-queries", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:hidden-sheet-names", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "msoffice:excel:protected-worksheet", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:sheet-number", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "msoffice:excel:sheet-numbers", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "msoffice:excel:very-hidden-sheet-names", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "msoffice:excel:workbook-codename", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-comments", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-external-chart-data", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-external-ole-objects", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-field-hyperlinks", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-hover-hyperlinks", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-linked-ole-objects", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-track-changes", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-unreferenced-parts", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:has-vml-hyperlinks", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:ocx-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "msoffice:ppt:has-animations", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:ppt:has-hidden-slides", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "msoffice:ppt:num-hidden-slides", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "msoffice:ppt:num-unlisted-slides", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "msoffice:ppt:unlisted-slide-names", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "msoffice:prog-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "msoffice:unreferenced-part-names", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "multipart:boundary", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "multipart:subtype", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "netcdf:file-type-description", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "numbers:sheet-names", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "ogg:codec-version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:buildNumberCreated", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:buildNumberLastWroteToFile", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:buildNumberNewestWritten", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:buildNumberOldestWritten", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:cTransactionsInLog", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:cbExpectedFileLength", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:cbFreeSpaceInFreeChunkList", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:cbLegacyExpectedFileLength", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:cbLegacyFreeSpaceInFreeChunkList", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:conflictPageGuids", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "onenote:crcName", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:creationTimestamp", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:ffvLastCodeThatWroteToThisFile", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:ffvNewestCodeThatHasWrittenToThisFile", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:ffvOldestCodeThatHasWrittenToThisFile", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:ffvOldestCodeThatMayReadThisFile", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:grfDebugLogFlags", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:lastModifiedTimestamp", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:mostRecentAuthors", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "onenote:nFileVersionGeneration", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:originalAuthors", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "onenote:pageGuids", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "onenote:pageNodeGuids", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "onenote:pageSeriesGuids", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "onenote:rgbPlaceholder", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "onenote:sectionGuid", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:action-trigger", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:action-triggers", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "pdf:action-types", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "pdf:annotation-subtypes", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "pdf:annotation-types", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "pdf:associated-file-relationship", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:chars-per-page", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "pdf:contains-damaged-font", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:contains-non-embedded-font", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:created", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:creator", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:creator-tool", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:keywords", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:modified", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:producer", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:subject", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:title", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:docinfo:trapped", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:embedded-file-annotation-type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:embedded-file-description", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:embedded-file-subtype", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:encrypted", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:eof-offsets", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "pdf:has-3d", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:has-acro-form-fields", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:has-collection", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:has-marked-content", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:has-signature-fields", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:has-xfa", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:has-xmp", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "pdf:illustrator:type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:incremental-update-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdf:incremental-update-number", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdf:js-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:marked-content-pages-fallback", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdf:marked-content-pages-tagged", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdf:marked-content-rejections", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "pdf:num-3d-annotations", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdf:ocr-page-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdf:overall-percentage-unmapped-unicode-chars", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "pdf:pdf-extension-version", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "pdf:pdf-version", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "pdf:producer", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:total-unmapped-unicode-chars", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdf:trapped", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdf:unmapped-unicode-chars-per-page", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "pdf:xmp-location", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdfa:pdf-version", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "pdfaid:conformance", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdfaid:part", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdfuaid:part", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "pdfvt:modified", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "pdfvt:version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdfx:conformance", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdfx:version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pdfxid:version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:AuthorsPosition", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:CaptionWriter", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:Category", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:City", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:ColorMode", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "photoshop:Country", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:Credit", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:DateCreated", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "photoshop:Headline", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:ICCProfile", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:Instructions", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:Source", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:State", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:SupplementalCategories", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "photoshop:TransmissionReference", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "photoshop:Urgency", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:CopyrightOwner", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:CopyrightOwnerID", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:CopyrightOwnerName", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:ImageCreator", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:ImageCreatorID", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:ImageCreatorName", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:ImageSupplier", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:ImageSupplierID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:ImageSupplierImageID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:ImageSupplierName", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:Licensor", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorCity", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorCountry", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorEmail", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorExtendedAddress", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorID", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorName", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorPostalCode", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorRegion", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorStreetAddress", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorTelephone1", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorTelephone2", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:LicensorURL", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:MinorModelAgeDisclosure", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:ModelReleaseID", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:ModelReleaseStatus", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:PropertyReleaseID", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "plus:PropertyReleaseStatus", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "plus:Version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pst:descriptor-node-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "pst:is-valid", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "quattropro:build", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "quattropro:id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "quattropro:lowest-version", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "quattropro:version", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "quicktime:still-image-time", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "rtf:contains-encapsulated-html", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "rtf:embedded-app-version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "rtf:embedded-class", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "rtf:embedded-item", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "rtf:embedded-topic", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "rtf:thumbnail", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "sqlite3:application_id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "sqlite3:user_version", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tess:image-magick-processed", "valueType": "BOOLEAN", "cardinality": "SEQ"}, + {"key": "tess:orientation", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tess:orientation-confidence", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "tess:page-number", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tess:rotate", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tess:rotation", "valueType": "REAL", "cardinality": "SEQ"}, + {"key": "tess:script", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tess:script-confidence", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "tiff:BitsPerSample", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "tiff:ImageLength", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tiff:ImageWidth", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tiff:Make", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tiff:Model", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tiff:Orientation", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "tiff:ResolutionUnit", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "tiff:SamplesPerPixel", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tiff:Software", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tiff:XResolution", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "tiff:YResolution", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "tk:chunks", "valueType": "JSON", "cardinality": "SIMPLE"}, + {"key": "tk:content", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:content-handler-type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:content-type-hint", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:content-type-magic-detected", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:content-type-override", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:content-type-parser-override", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:decoded-charset", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:detected-encoding", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:detected-language", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:detected-language-confidence", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:detected-language-confidence-raw", "valueType": "REAL", "cardinality": "SEQ"}, + {"key": "tk:detection-content-length", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:digest:MD2", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:MD2:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:MD2:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:MD5", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:MD5:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:MD5:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-1", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-1:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-1:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-256", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-256:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-256:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-384", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-384:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-384:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-512", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-512:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA-512:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-256", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-256:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-256:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-384", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-384:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-384:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-512", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-512:BASE32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:digest:SHA3-512:BASE64", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:embedded-depth", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:embedded-id", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:embedded-id-path", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:embedded-relationship-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:embedded-resource-path", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:embedded-resource-type", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "tk:encoding-detection-trace", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:encoding-detector", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:encrypted", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:exception:container-exception", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:exception:embedded-bytes-exception", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:exception:embedded-depth-limit-reached", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:exception:embedded-exception", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:exception:embedded-parser", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:exception:embedded-resource-limit-reached", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:exception:embedded-stream-exception", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:exception:embedded-warning", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:exception:task-deadline-reached", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:exception:warn", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:exception:write-limit-reached", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:final-embedded-resource-path", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:has-signature", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:inference-released", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:internal-path", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:num-images", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:ocr-warnings", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:orig-resource-name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:page:number", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:page:numbers", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "tk:page:rotation", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "tk:parse-time-millis", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:parsed-by", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:parsed-by-full-set", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:pipes-result", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:rendering:failed-page", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "tk:rendering:pdfbox-image-writing-ms", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "tk:rendering:pdfbox-rendering-ms", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "tk:rendering:rendered-by", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:rendering:rendering-time-ms", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "tk:resource-name", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:resource-name-extension-inferred", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:signature:contact-info", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:signature:date", "valueType": "DATE", "cardinality": "BAG"}, + {"key": "tk:signature:filter", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:signature:location", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:signature:name", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:signature:reason", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:source-path", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "tk:truncated-content-for-detection", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:version-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:version-number", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:warn:truncated-metadata", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "tk:xml-external-reference", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "tk:xml-external-reference-count", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "tk:xml-external-reference-embedded", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "video:bitrate", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "video:fourcc", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "video:frame-rate", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "vlm:completion-tokens", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "vlm:image-pixels", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "vlm:model", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "vlm:prompt-tokens", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "vlm:skipped-reason", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "vorbis:vendor", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "w:Comments", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "warc:WARC-Record-ID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "warc:payload-content-type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "warc:record-content-type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "warc:warning", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "wordperfect:encrypted", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "wordperfect:file-id", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "wordperfect:file-size", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "wordperfect:file-type", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "wordperfect:major-version", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "wordperfect:minor-version", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "wordperfect:product-type", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmlprofiler:entity-local-names", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmlprofiler:entity-uris", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmlprofiler:root-entity", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:About", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:Advisory", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:CreateDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmp:CreatorTool", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:Identifier", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:Label", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:MetadataDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmp:ModifyDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmp:NickName", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:Rating", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmp:Title", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:contributor", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:dc:coverage", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:creator", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:dc:date", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:description", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:format", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:identifier", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:language", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:publisher", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:relation", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:rights", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:source", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:subject", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:dc:title", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dc:type", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:dcterms:created", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmp:dcterms:modified", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmp:exif:DateTimeOriginal", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmp:exif:IsoSpeedRatings", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "xmp:exif:PixelXDimension", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmp:exif:PixelYDimension", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmp:pdf:About", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:pdf:Keywords", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmp:pdf:PDFVersion", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:pdf:Producer", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:pdf:Trapped", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:tiff:BitsPerSample", "valueType": "INTEGER", "cardinality": "SEQ"}, + {"key": "xmp:tiff:ImageLength", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmp:tiff:ImageWidth", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmp:tiff:Make", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:tiff:Model", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:tiff:Orientation", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmp:tiff:SamplesPerPixel", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmp:tiff:Software", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:absPeakAudioFilePath", "valueType": "URI", "cardinality": "SIMPLE"}, + {"key": "xmpDM:album", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:albumArtist", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:altTapeName", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:artist", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:audioChannelType", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:audioCompressor", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:audioModDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:audioSampleRate", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmpDM:audioSampleType", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:compilation", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmpDM:composer", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:copyright", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:discNumber", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmpDM:duration", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "xmpDM:engineer", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:fileDataRate", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "xmpDM:genre", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:instrument", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:key", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:logComment", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpDM:loop", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "xmpDM:metadataModDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:numberOfBeats", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "xmpDM:pullDown", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:relativePeakAudioFilePath", "valueType": "URI", "cardinality": "SIMPLE"}, + {"key": "xmpDM:releaseDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:scaleType", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:scene", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:shotDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:shotLocation", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:shotName", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:speakerPlacement", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:stretchMode", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:tapeName", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:tempo", "valueType": "REAL", "cardinality": "SIMPLE"}, + {"key": "xmpDM:timeSignature", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:trackNumber", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoAlphaMode", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoAlphaUnityIsTransparent", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoColorSpace", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoCompressor", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoFieldOrder", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoFrameRate", "valueType": "OPEN_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoModDate", "valueType": "DATE", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoPixelAspectRatio", "valueType": "RATIONAL", "cardinality": "SIMPLE"}, + {"key": "xmpDM:videoPixelDepth", "valueType": "CLOSED_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpMM:DerivedFrom:DocumentID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpMM:DerivedFrom:InstanceID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpMM:DerivedFrom:OriginalDocumentID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpMM:DerivedFrom:RenditionClass", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpMM:DocumentID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpMM:History:Action", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpMM:History:Changed", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpMM:History:InstanceID", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpMM:History:Parameters", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpMM:History:SoftwareAgent", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpMM:History:When", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpMM:InstanceID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpMM:OriginalDocumentID", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpMM:RenditionClass", "valueType": "OPEN_CHOICE", "cardinality": "SIMPLE"}, + {"key": "xmpMM:RenditionParams", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpRights:Certificate", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpRights:Marked", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "xmpRights:Owner", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "xmpRights:UsageTerms", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpRights:WebStatement", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "xmpTPg:NPages", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "xmpidq:Scheme", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "zip:central-directory-only-entries", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "zip:comment", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "zip:compressed-size", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "zip:compression-method", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "zip:crc32", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "zip:detector-data-descriptor-required", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "zip:detector-zip-file-opened", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "zip:duplicate-entry-names", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "zip:encrypted", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "zip:integrity-check-result", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "zip:local-header-only-entries", "valueType": "TEXT", "cardinality": "BAG"}, + {"key": "zip:platform", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "zip:salvaged", "valueType": "BOOLEAN", "cardinality": "SIMPLE"}, + {"key": "zip:uncompressed-size", "valueType": "TEXT", "cardinality": "SIMPLE"}, + {"key": "zip:unix-mode", "valueType": "INTEGER", "cardinality": "SIMPLE"}, + {"key": "zip:version-made-by", "valueType": "INTEGER", "cardinality": "SIMPLE"} +] diff --git a/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/type-changes.json b/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/type-changes.json new file mode 100644 index 00000000000..0d4f101c7a3 --- /dev/null +++ b/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/schema/type-changes.json @@ -0,0 +1,2 @@ +[ +]