[WIP] Metastore contextual search

packaging/pom.xml

@@ -453,6 +453,11 @@
       <artifactId>hive-standalone-metastore-rest-catalog</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-standalone-metastore-search</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs-client</artifactId>

packaging/src/main/assembly/src.xml

@@ -106,6 +106,7 @@
         <include>storage-api/**/*</include>
         <include>standalone-metastore/metastore-common/**/*</include>
         <include>standalone-metastore/metastore-client/**/*</include>
+        <include>standalone-metastore/metastore-search/**/*</include>
         <include>standalone-metastore/metastore-server/**/*</include>
         <include>standalone-metastore/metastore-tools/**/*</include>
         <include>standalone-metastore/metastore-rest-catalog/**/*</include>

standalone-metastore/metastore-search/pom.xml

@@ -0,0 +1,219 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>hive-standalone-metastore</artifactId>
+    <groupId>org.apache.hive</groupId>
+    <version>4.3.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>hive-standalone-metastore-search</artifactId>
+  <name>Hive Metastore Search</name>
+  <properties>
+    <standalone.metastore.path.to.root>..</standalone.metastore.path.to.root>
+    <onnxruntime.version>1.25.1</onnxruntime.version>
+    <djl.tokenizers.version>0.36.0</djl.tokenizers.version>
+    <lucene.version>10.5.0</lucene.version>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-standalone-metastore-server</artifactId>
+      <version>${hive.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-reload4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>ch.qos.reload4j</groupId>
+          <artifactId>reload4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs-client</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-reload4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>ch.qos.reload4j</groupId>
+          <artifactId>reload4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.microsoft.onnxruntime</groupId>
+      <artifactId>onnxruntime</artifactId>
+      <version>${onnxruntime.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>ai.djl.huggingface</groupId>
+      <artifactId>tokenizers</artifactId>
+      <version>${djl.tokenizers.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-core</artifactId>
+      <version>${lucene.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analysis-common</artifactId>
+      <version>${lucene.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-queryparser</artifactId>
+      <version>${lucene.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-standalone-metastore-server</artifactId>
+      <version>${hive.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.derby</groupId>
+      <artifactId>derby</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive.hcatalog</groupId>
+      <artifactId>hive-hcatalog-server-extensions</artifactId>
+      <version>${hive.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>setup-test-dirs</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <delete dir="${test.tmp.dir}"/>
+                <delete dir="${test.warehouse.external.dir}"/>
+                <delete dir="${test.warehouse.dir}"/>
+                <mkdir dir="${test.tmp.dir}"/>
+                <mkdir dir="${test.warehouse.dir}"/>
+                <mkdir dir="${test.warehouse.external.dir}"/>
+              </target>
+            </configuration>
+          </execution>
+          <execution>
+            <id>setup-metastore-scripts</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <mkdir dir="${test.tmp.dir}/scripts/metastore/upgrade"/>
+                <copy todir="${test.tmp.dir}/scripts/metastore/upgrade">
+                  <fileset dir="${basedir}/../metastore-server/src/main/sql/"/>
+                </copy>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <redirectTestOutputToFile>true</redirectTestOutputToFile>
+          <reuseForks>false</reuseForks>
+          <argLine>-Xmx2048m --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.regex=ALL-UNNAMED --add-opens java.sql/java.sql=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.text=ALL-UNNAMED</argLine>
+          <systemPropertyVariables>
+            <build.dir>${project.build.directory}</build.dir>
+            <datanucleus.schema.autoCreateAll>true</datanucleus.schema.autoCreateAll>
+            <derby.version>${derby.version}</derby.version>
+            <derby.stream.error.file>${test.tmp.dir}/derby.log</derby.stream.error.file>
+            <java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>
+            <javax.jdo.option.ConnectionURL>jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true</javax.jdo.option.ConnectionURL>
+            <metastore.schema.verification>false</metastore.schema.verification>
+            <test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
+            <metastore.warehouse.dir>${test.warehouse.scheme}${test.warehouse.dir}</metastore.warehouse.dir>
+            <hive.metastore.warehouse.external.dir>${test.warehouse.scheme}${test.warehouse.external.dir}</hive.metastore.warehouse.external.dir>
+          </systemPropertyVariables>
+          <additionalClasspathElements>
+            <additionalClasspathElement>${project.build.testOutputDirectory}</additionalClasspathElement>
+          </additionalClasspathElements>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

standalone-metastore/metastore-search/src/main/java/org/apache/hadoop/hive/metastore/handler/SearchProvider.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.handler;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.search.exception.IndexNotReadyException;
+import org.apache.hive.search.exception.InitializeException;
+import org.apache.hive.search.exception.SearchException;
+import org.apache.hive.search.search.LuceneSearchBackend;
+import org.apache.hive.search.search.SearchBackend;
+import org.apache.hive.search.search.SearchQuery;
+import org.apache.hive.search.search.TableSearchResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Single server-side provider that owns the lifecycle of the {@link SearchBackend}. */
+public final class SearchProvider implements AutoCloseable {
+  private static final Logger LOG = LoggerFactory.getLogger(SearchProvider.class);
+  private static final AtomicReference<SearchProvider> INSTANCE = new AtomicReference<>();
+
+  private final Configuration configuration;
+  private final SearchBackend backend;
+
+  private SearchProvider(Configuration configuration, SearchBackend backend) {
+    this.configuration = configuration;
+    this.backend = backend;
+  }
+
+  /** Creates and initializes a provider with the supplied backend implementation. */
+  private static SearchProvider create(Configuration configuration, SearchBackend backend)
+      throws InitializeException, IOException {
+    Objects.requireNonNull(configuration, "configuration");
+    Objects.requireNonNull(backend, "backend");
+    backend.initialize(configuration);
+    LOG.info("Installed search backend {}", backend.getClass().getSimpleName());
+    return new SearchProvider(configuration, backend);
+  }
+
+  public static SearchProvider install(Configuration configuration)
+      throws InitializeException, IOException {
+    return install(configuration, new LuceneSearchBackend());
+  }
+
+  /** Installs the process-wide provider using the supplied backend implementation. */
+  public static SearchProvider install(Configuration configuration, SearchBackend backend)
+      throws InitializeException, IOException {
+    SearchProvider provider;
+    if ((provider = INSTANCE.get()) != null) {
+      return provider;
+    }
+    synchronized (SearchProvider.class) {
+      if ((provider = INSTANCE.get()) == null) {
+        provider = create(configuration, backend);
+        INSTANCE.getAndSet(provider);
+      }
+    }
+    return provider;
+  }
+
+  /** Returns the installed provider. */
+  public static SearchProvider get() {
+    SearchProvider provider = INSTANCE.get();
+    if (provider == null) {
+      throw new IllegalStateException("SearchProvider is not installed");
+    }
+    return provider;
+  }
+
+  /** Clears the installed provider; intended for tests. */
+  public static void reset() throws Exception {
+    SearchProvider previous = INSTANCE.getAndSet(null);
+    if (previous != null) {
+      previous.close();
+    }
+  }
+
+  public Configuration configuration() {
+    return configuration;
+  }
+
+  public SearchBackend backend() {
+    return backend;
+  }
+
+  public boolean isReady() throws IndexNotReadyException {
+    return backend.isReady();
+  }
+
+  public TableSearchResult search(SearchQuery query)
+      throws SearchException, InitializeException, IOException {
+    return backend.search(query);
+  }
+
+  @Override
+  public void close() throws Exception {
+    backend.close();
+  }
+}