Prioritize target classpath over Turbine headers

jsemanticdb/src/main/protobuf/mbt.proto

@@ -36,6 +36,7 @@ message IndexedDocument {
     V7 = 7;
     V8 = 8;
     V9 = 9;
+    V10 = 10;
   }
 
   enum Source {

metals/src/main/scala/scala/meta/internal/metals/mbt/IndexedDocument.scala

@@ -96,7 +96,7 @@ case class IndexedDocument(
   def toProto(): Mbt.IndexedDocument.Builder = {
     val bloomFilterVersion = language match {
       case Language.JAVA => Mbt.IndexedDocument.BloomFilterVersion.V7
-      case Language.PROTOBUF => Mbt.IndexedDocument.BloomFilterVersion.V9
+      case Language.PROTOBUF => Mbt.IndexedDocument.BloomFilterVersion.V10
       case _ => Mbt.IndexedDocument.BloomFilterVersion.V6
     }
     Mbt.IndexedDocument
@@ -271,8 +271,8 @@ object IndexedDocument {
     doc.getBloomFilterVersion().getNumber >= (doc.getLanguage() match {
       case Language.JAVA => Mbt.IndexedDocument.BloomFilterVersion.V7
       case Language.SCALA => Mbt.IndexedDocument.BloomFilterVersion.V6
-      // V9: Proto bloom filters include scanner fixes for option blocks.
-      case Language.PROTOBUF => Mbt.IndexedDocument.BloomFilterVersion.V9
+      // V10: Includes V9 option-block scanner fixes and Java packages for Turbine.
+      case Language.PROTOBUF => Mbt.IndexedDocument.BloomFilterVersion.V10
       case _ => Mbt.IndexedDocument.BloomFilterVersion.V1
     }).getNumber()
 }

metals/src/main/scala/scala/meta/internal/metals/mbt/MbtWorkspaceSymbolProvider.scala

@@ -136,6 +136,10 @@ class MbtWorkspaceSymbolProvider(
     clearAllProtobufCaches,
   )
 
+  private def isProtoJavaPackageIndexingEnabled: Boolean =
+    protobufWorkspace.isJavaPackageIndexingEnabled ||
+      javaSymbolLoader().isTurbineClasspath
+
   /**
    * The Java outlines synthesized from the given `.proto` file (one per
    * generated top-level class). Empty when the file isn't an indexed proto or
@@ -202,10 +206,7 @@ class MbtWorkspaceSymbolProvider(
               )
             )
             .toList
-        } else if (
-          file.isProtoFilename &&
-          protobufWorkspace.isJavaPackageIndexingEnabled
-        ) {
+        } else if (file.isProtoFilename && isProtoJavaPackageIndexingEnabled) {
           // Include the Java outlines generated from proto files so that
           // turbine can resolve references to proto-generated classes when it
           // header-compiles the workspace. Without these, a Java method
@@ -547,7 +548,7 @@ class MbtWorkspaceSymbolProvider(
   ): Future[Unit] = try {
     if (MbtIndexFilter.included(indexFilters, MbtFileCandidate(file))) {
       val enableProtoJavaPackage =
-        file.isProtoFilename && protobufWorkspace.isJavaPackageIndexingEnabled
+        file.isProtoFilename && isProtoJavaPackageIndexingEnabled
       val mdoc =
         IndexedDocument.fromFile(
           file,

metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineClasspathFileManager.scala

@@ -21,6 +21,7 @@ class TurbineClasspathFileManager(
     delegate: JavaFileManager,
     workspaceClasspath: () => TurbineCompileResult,
     listSourcepath: String => java.lang.Iterable[JavaFileObject],
+    listProtoBinaryNames: String => Set[String],
     isDeleted: String => Boolean,
     projectClasspath: ClassPath,
 ) extends ForwardingJavaFileManager[JavaFileManager](delegate) {
@@ -83,6 +84,20 @@ class TurbineClasspathFileManager(
         val turbinePackageName = packageNames.mkString("/")
         val objects = new ju.ArrayList[JavaFileObject]()
         val cp = workspaceClasspath()
+        val isAddedBinaryName = new ju.HashSet[String]()
+        val protoBinaryNames = listProtoBinaryNames(packageName)
+        if (protoBinaryNames.nonEmpty) {
+          super.list(location, packageName, kinds, recurse).forEach { obj =>
+            val binaryName = inferBinaryName(location, obj).replace('.', '/')
+            val topLevelBinaryName = binaryName.takeWhile(_ != '$')
+            if (
+              protoBinaryNames.contains(topLevelBinaryName) &&
+              isAddedBinaryName.add(binaryName)
+            ) {
+              objects.add(obj)
+            }
+          }
+        }
         cp.symbolsByPackage.get(turbinePackageName) match {
           case None =>
           case Some(values) =>
@@ -94,7 +109,7 @@ class TurbineClasspathFileManager(
               // or have a pending source on SOURCE_PATH (so javac uses the updated source)
               if (!isDeleted(binaryName)) {
                 val bytes = cp.lowered.bytes().get(binaryName)
-                if (bytes != null) {
+                if (bytes != null && isAddedBinaryName.add(binaryName)) {
                   val obj = new TurbineClassfileObject(
                     binaryName,
                     () => bytes,
@@ -104,14 +119,13 @@ class TurbineClasspathFileManager(
               }
             }
         }
-        val isAddedBinaryName = new ju.HashSet[String]()
         for {
-          cp <- List(
-            // Prioritize the project classpath over the fallback classpath
-            projectClasspath,
-            cp.classpath,
-          )
-        } listPackageClasspath(cp, packageNames, isAddedBinaryName) { obj =>
+          classpath <- List(projectClasspath, cp.classpath)
+        } listPackageClasspath(
+          classpath,
+          packageNames,
+          isAddedBinaryName,
+        ) { obj =>
           objects.add(obj)
         }
         objects

metals/src/main/scala/scala/meta/internal/metals/mbt/TurbineCompiler.scala

@@ -24,6 +24,7 @@ import scala.meta.internal.metals.PcQueryContext
 import scala.meta.internal.metals.ReportContext
 import scala.meta.internal.metals.Sleeper
 import scala.meta.pc.ProgressBars
+import scala.meta.pc.SemanticdbCompilationUnit
 
 import com.google.common.collect.ImmutableList
 import com.google.common.collect.ImmutableMap
@@ -329,17 +330,18 @@ class TurbineCompiler[T](
       projectClasspathJars: ju.List[Path],
   ): JavaFileManager = {
     val isGlobalClasspathEntry = this.classpath().toSet
-    val filteredProjectClasspath =
+    val projectClasspathEntries =
       projectClasspathJars.asScala.filter(file =>
         !isGlobalClasspathEntry(file) && TurbineCompiler.isJarFile(file)
       )
     val projectClasspath =
-      ClassPathBinder.bindClasspath(filteredProjectClasspath.asJava)
+      ClassPathBinder.bindClasspath(projectClasspathEntries.asJava)
     onNewProjectClasspath(projectClasspath)
     new TurbineClasspathFileManager(
       underlying,
       () => result,
       listSourcepath = listCombinedSourcepath,
+      listProtoBinaryNames,
       isDeleted,
       projectClasspath,
     )
@@ -362,6 +364,14 @@ class TurbineCompiler[T](
     }
   }
 
+  private def listProtoBinaryNames(packageName: String): Set[String] = {
+    val protoPackage = packageName.replace('.', '/') + "/"
+    listProtoJavaOutlinesForPackage(protoPackage).collect {
+      case file: SemanticdbCompilationUnit =>
+        file.binaryName().replace('.', '/')
+    }.toSet
+  }
+
   private[mbt] def listSourcepath(
       packageName: String
   ): java.lang.Iterable[JavaFileObject] = {

tests/unit/src/test/scala/tests/mbt/MbtWorkspaceSymbolProviderSuite.scala

@@ -1,17 +1,35 @@
 package tests.mbt
 
+import java.net.URI
 import java.nio.file.Files
 import java.nio.file.Paths
+import java.util.EnumSet
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import javax.tools.JavaFileObject
+import javax.tools.StandardLocation
+import javax.tools.ToolProvider
 
+import scala.collection.JavaConverters._
+import scala.collection.parallel.mutable.ParArray
 import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
+import scala.util.Using
 
 import scala.meta.internal.metals.Configs
+import scala.meta.internal.metals.EmptyWorkDoneProgress
+import scala.meta.internal.metals.LoggerReportContext
+import scala.meta.internal.metals.Sleeper
 import scala.meta.internal.metals.mbt.IndexingStats
 import scala.meta.internal.metals.mbt.MbtWorkspaceSymbolProvider
+import scala.meta.internal.metals.mbt.TurbineCompileResult
+import scala.meta.internal.metals.mbt.TurbineCompiler
+import scala.meta.internal.metals.mbt.VirtualTextDocument
 import scala.meta.io.AbsolutePath
+import scala.meta.pc
 
+import com.google.turbine.diag.SourceFile
 import munit.AnyFixture
 import munit.TestOptions
 import org.eclipse.{lsp4j => l}
@@ -154,6 +172,27 @@ object Hello2 {
     )
   }
 
+  test("turbine-indexes-proto-java-package-with-protobuf-lsp-disabled") {
+    FileLayout.fromString(
+      """
+/example/Dependency.proto
+syntax = "proto3";
+package example;
+option java_package = "generated.example";
+message Dependency {}
+""",
+      root = workspace(),
+    )
+    val provider = newProvider()
+    workspace.executeCommand("git init -b main")
+    workspace.gitCommitAllChanges()
+    assertEquals(
+      provider.onReindex().awaitBackgroundJobs(),
+      IndexingStats(totalFiles = 1, updatedFiles = 1),
+    )
+    assert(provider.listAllPackages().containsKey("generated/example/"))
+  }
+
   test("exclude-module-info-java") {
     FileLayout.fromString(
       """
@@ -227,3 +266,112 @@ module com.example {
   )
 
 }
+
+class TurbineClasspathFileManagerSuite extends munit.FunSuite {
+  implicit val reportContext: LoggerReportContext.type = LoggerReportContext
+  implicit val executionContext: ExecutionContext = ExecutionContext.global
+
+  private val workspaceSource =
+    "package example; public class Dependency { public static class Builder { public void workspace() {} } }"
+  private val projectSource =
+    "package example; public class Dependency { public static class Builder { public void project() {} } }"
+
+  private def compile(source: String): TurbineCompileResult =
+    TurbineCompiler.compileClassfiles(
+      ParArray(source),
+      (text: String) => Seq(new SourceFile("Dependency.java", text)),
+      Nil,
+      EmptyWorkDoneProgress,
+    )
+
+  private def checkTargetClasspathPrecedence(isProtobuf: Boolean): Unit = {
+    val projectResult = compile(projectSource)
+    val jar = Files.createTempFile("metals-project-classpath", ".jar")
+    val output = new ZipOutputStream(Files.newOutputStream(jar))
+    try {
+      for ((name, bytes) <- projectResult.lowered.bytes().asScala) {
+        output.putNextEntry(new ZipEntry(s"$name.class"))
+        output.write(bytes)
+        output.closeEntry()
+      }
+    } finally output.close()
+
+    var fallbackClasspath = Seq.empty[java.nio.file.Path]
+    val protoOutline = VirtualTextDocument(
+      URI.create("file:///Dependency.java"),
+      pc.Language.JAVA,
+      workspaceSource,
+      Seq("example"),
+      Seq("example/Dependency#"),
+    )
+    val compiler = new TurbineCompiler[String](
+      () => ParArray(workspaceSource),
+      text => Seq(new SourceFile("Dependency.java", text)),
+      () => fallbackClasspath,
+      EmptyWorkDoneProgress,
+      () => Configs.TurbineRecompileDelayConfig.testing,
+      packageName =>
+        if (isProtobuf && packageName == "example/") Iterator(protoOutline)
+        else Iterator.empty,
+      Sleeper.TestingSleeper,
+      () => (),
+      _ => (),
+    )
+    compiler.doCompileNow()
+    val workspaceResult = compiler.result
+    fallbackClasspath = Seq(jar)
+
+    val standardFileManager = ToolProvider
+      .getSystemJavaCompiler()
+      .getStandardFileManager(null, null, null)
+    standardFileManager.setLocationFromPaths(
+      StandardLocation.CLASS_PATH,
+      List(jar).asJava,
+    )
+    val fileManager = compiler.createFileManager(
+      standardFileManager,
+      List(jar).asJava,
+    )
+    try {
+      val classfiles = fileManager
+        .list(
+          StandardLocation.CLASS_PATH,
+          "example",
+          EnumSet.of(JavaFileObject.Kind.CLASS),
+          false,
+        )
+        .asScala
+        .toList
+      val obtained = classfiles.map { classfile =>
+        val binaryName = fileManager
+          .inferBinaryName(StandardLocation.CLASS_PATH, classfile)
+          .replace('.', '/')
+        binaryName -> Using.resource(classfile.openInputStream())(
+          _.readAllBytes().toSeq
+        )
+      }.toMap
+      val expectedResult =
+        if (isProtobuf) projectResult
+        else workspaceResult
+      val expected = expectedResult.lowered
+        .bytes()
+        .asScala
+        .map { case (name, bytes) =>
+          name -> bytes.toSeq
+        }
+        .toMap
+      assertEquals(obtained, expected)
+    } finally {
+      fileManager.close()
+      Files.deleteIfExists(jar)
+    }
+  }
+
+  test("target-protobuf-classpath-before-workspace-headers") {
+    checkTargetClasspathPrecedence(isProtobuf = true)
+  }
+
+  test("workspace-headers-before-non-protobuf-target-classpath") {
+    checkTargetClasspathPrecedence(isProtobuf = false)
+  }
+}