[Bazel] Support for a custom prebuilt emscripten cache

bazel/README.md

@@ -87,3 +87,23 @@ emscripten_cache.targets(targets = [
 ```
 
 See `test_external/` for an example using [embind](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html).
+
+Alternatively, you can use the embuilder to build the cache manually and put it into
+an archive that you serve from your HTTP server. Then you can declare it in your
+`MODULE.bazel` as follows:
+
+```starlark
+
+emscripten_cache = use_extension(
+    "@emsdk//:emscripten_cache.bzl",
+    "emscripten_cache",
+)
+
+emscripten_cache.prebuilt_cache(
+    http_archive_url = "https://my-host.com/my-emsdk-cache-4.0.16.tar.gz",
+    sha256 = "3e88abcbd22bac7b05af416c8f1859d12572c8e9356db604a2768fcfda863da8",
+    strip_prefix = "my-emsdk-cache",
+)
+```
+
+You cannot use both `prebuilt_cache` and `configuration`/`targets` at the same time. If you try to do so, `prebuilt_cache` will take precedence.

bazel/emscripten_build_file.bzl

@@ -18,6 +18,13 @@ filegroup(
     ]),
 )
 
+filegroup(
+    name = "builtin_cache",
+    srcs = glob([
+        "emscripten/cache/**",
+    ]),
+)
+
 filegroup(
     name = "emcc_common",
     srcs = [

bazel/emscripten_cache.bzl

@@ -3,6 +3,21 @@ package(default_visibility = ['//visibility:public'])
 exports_files(['emscripten_config'])
 """
 
+BUILD_FILE_USE_BUILTIN_CACHE = """
+alias(
+    name = "emscripten_cache",
+    actual = "{}//:builtin_cache",
+)
+"""
+
+BUILD_FILE_USE_SECONDARY_CACHE = """
+filegroup(
+    name = "emscripten_cache",
+    srcs = glob(["cache/**"]),
+    visibility = ["//visibility:public"],
+)
+"""
+
 EMBUILDER_CONFIG_TEMPLATE = """
 CACHE = '{cache}'
 BINARYEN_ROOT = '{binaryen_root}'
@@ -29,6 +44,26 @@ def get_root_and_script_ext(repository_ctx):
     else:
         fail("Unsupported operating system")
 
+def get_bin_deps_repo_name(repository_ctx):
+    if repository_ctx.os.name.startswith("linux"):
+        if "amd64" in repository_ctx.os.arch or "x86_64" in repository_ctx.os.arch:
+            return "@emscripten_bin_linux"
+        elif "aarch64" in repository_ctx.os.arch:
+            return "@emscripten_bin_linux_arm64"
+        else:
+            fail("Unsupported architecture for Linux")
+    elif repository_ctx.os.name.startswith("mac"):
+        if "amd64" in repository_ctx.os.arch or "x86_64" in repository_ctx.os.arch:
+            return "@emscripten_bin_mac"
+        elif "aarch64" in repository_ctx.os.arch:
+            return "@emscripten_bin_mac_arm64"
+        else:
+            fail("Unsupported architecture for MacOS")
+    elif repository_ctx.os.name.startswith("windows"):
+        return "@emscripten_bin_win"
+    else:
+        fail("Unsupported operating system")
+
 def _emscripten_cache_repository_impl(repository_ctx):
     # Read the default emscripten configuration file
     default_config = repository_ctx.read(
@@ -37,7 +72,26 @@ def _emscripten_cache_repository_impl(repository_ctx):
         ),
     )
 
-    if repository_ctx.attr.targets or repository_ctx.attr.configuration:
+    repo_metadata = None
+    build_file_content = BUILD_FILE_CONTENT_TEMPLATE
+    use_builtin_cache = True
+
+    if repository_ctx.attr.prebuilt_cache_url:
+        repository_ctx.download_and_extract(
+            url = repository_ctx.attr.prebuilt_cache_url,
+            output = "cache",
+            sha256 = repository_ctx.attr.prebuilt_cache_sha256,
+            stripPrefix = repository_ctx.attr.prebuilt_cache_strip_prefix,
+        )
+
+        # Use the prebuilt cache
+        use_builtin_cache = False
+
+        # Bazel 7 does not have the repo_metadata API, so prebuilt cache on Bazel 7 will not be marked as reproducible. This is not ideal, but it is a limitation of Bazel 7.
+        if hasattr(repository_ctx, "repo_metadata"):
+            repo_metadata = repository_ctx.repo_metadata(reproducible = True)
+
+    elif repository_ctx.attr.targets or repository_ctx.attr.configuration:
         root, script_ext = get_root_and_script_ext(repository_ctx)
         llvm_root = root.get_child("bin")
         cache = repository_ctx.path("cache")
@@ -72,42 +126,72 @@ def _emscripten_cache_repository_impl(repository_ctx):
             },
         )
         if result.return_code != 0:
-            fail("Embuilder exited with a non-zero return code")
+            fail("Embuilder exited with a non-zero return code\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr))
 
-        # Override Emscripten's cache with the secondary cache
-        default_config += "CACHE = '{}'\n".format(cache)
+        use_builtin_cache = False
+
+    if use_builtin_cache:
+        build_file_content += BUILD_FILE_USE_BUILTIN_CACHE.format(get_bin_deps_repo_name(repository_ctx))
+    else:
+        default_config += 'CACHE = os.path.join(os.path.dirname(os.environ["EM_CONFIG_PATH"]), "cache")\n'
+        build_file_content += BUILD_FILE_USE_SECONDARY_CACHE
 
     # Create the configuration file for the toolchain and export
     repository_ctx.file("emscripten_config", default_config)
-    repository_ctx.file("BUILD.bazel", BUILD_FILE_CONTENT_TEMPLATE)
+    repository_ctx.file("BUILD.bazel", build_file_content)
+
+    return repo_metadata
 
 _emscripten_cache_repository = repository_rule(
     implementation = _emscripten_cache_repository_impl,
     attrs = {
         "configuration": attr.string_list(),
         "targets": attr.string_list(),
+        "prebuilt_cache_url": attr.string(),
+        "prebuilt_cache_sha256": attr.string(),
+        "prebuilt_cache_strip_prefix": attr.string(),
     },
 )
 
 def _emscripten_cache_impl(ctx):
     all_configuration = []
     all_targets = []
+
+    prebuilt_cache_url = ""
+    prebuilt_cache_sha256 = ""
+    prebuilt_cache_strip_prefix = ""
+    prebuilt_cache_seen = False
     for mod in ctx.modules:
         for configuration in mod.tags.configuration:
             all_configuration += configuration.flags
         for targets in mod.tags.targets:
             all_targets += targets.targets
+        for prebuilt_cache in mod.tags.prebuilt_cache:
+            if prebuilt_cache_seen:
+                fail("Only one prebuilt_cache tag is allowed")
+            prebuilt_cache_url = prebuilt_cache.http_archive_url
+            prebuilt_cache_sha256 = prebuilt_cache.sha256
+            prebuilt_cache_strip_prefix = prebuilt_cache.strip_prefix
+            prebuilt_cache_seen = True
 
     _emscripten_cache_repository(
         name = "emscripten_cache",
         configuration = all_configuration,
         targets = all_targets,
+        prebuilt_cache_url = prebuilt_cache_url,
+        prebuilt_cache_sha256 = prebuilt_cache_sha256,
+        prebuilt_cache_strip_prefix = prebuilt_cache_strip_prefix,
     )
 
 emscripten_cache = module_extension(
     tag_classes = {
         "configuration": tag_class(attrs = {"flags": attr.string_list()}),
         "targets": tag_class(attrs = {"targets": attr.string_list()}),
+        "prebuilt_cache": tag_class(attrs = {
+            "http_archive_url": attr.string(mandatory = True),
+            "sha256": attr.string(mandatory = True),
+            "strip_prefix": attr.string(),
+        }),
     },
     implementation = _emscripten_cache_impl,
 )

bazel/remote_emscripten_repository.bzl

@@ -63,6 +63,7 @@ def create_toolchains(name, repo_name, exec_compatible_with):
         name = common_files_name,
         srcs = [
             "@emscripten_cache//:emscripten_config",
+            "@emscripten_cache//:emscripten_cache",
             "@emsdk//emscripten_toolchain:env.sh",
             "@emsdk//emscripten_toolchain:env.bat",
             "@rules_nodejs//nodejs:current_node_toolchain",

bazel/test_prebuilt_cache/.bazelrc

@@ -0,0 +1 @@
+build --incompatible_enable_cc_toolchain_resolution

bazel/test_prebuilt_cache/.gitignore

@@ -0,0 +1,4 @@
+bazel-bin
+bazel-out
+bazel-test_prebuilt_cache
+bazel-testlogs

bazel/test_prebuilt_cache/BUILD

@@ -0,0 +1,23 @@
+load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary")
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
+cc_binary(
+    name = "hello-world",
+    srcs = ["hello-world.cc"],
+    copts = [
+        "-flto=thin",
+    ],
+    linkopts = [
+        "-sAUTO_NATIVE_LIBRARIES=0",
+        "-flto=thin",
+    ],
+)
+
+wasm_cc_binary(
+    name = "hello-world-wasm",
+    cc_target = ":hello-world",
+    outputs = [
+        "hello-world.js",
+        "hello-world.wasm",
+    ],
+)

bazel/test_prebuilt_cache/MODULE.bazel

@@ -0,0 +1,29 @@
+bazel_dep(name = "rules_cc", version = "0.2.16")
+bazel_dep(name = "emsdk")
+local_path_override(
+    module_name = "emsdk",
+    path = "..",
+)
+
+emscripten_deps = use_extension(
+    "@emsdk//:emscripten_deps.bzl",
+    "emscripten_deps",
+)
+
+# Need to use the same version of Emscripten as the prebuilt cache was built with to ensure compatibility even when future versions of Emscripten are released.
+emscripten_deps.config(
+    version = "5.0.7",
+)
+
+emscripten_cache = use_extension(
+    "@emsdk//:emscripten_cache.bzl",
+    "emscripten_cache",
+)
+
+# This cache was built with Emscripten 5.0.7, and contains opt-thinlto build.
+# It has been build according to these instructions: https://github.com/DoDoENT/bazel-playground/blob/master/tools/emsdk-cache-builder/README.md
+emscripten_cache.prebuilt_cache(
+    http_archive_url = "https://github.com/DoDoENT/bazel-playground/releases/download/emsdk-cache/emsdk-cache-5.0.7.tar.gz",
+    sha256 = "bbfdab09ae64769c4aa977b1d88f776490c459c87e7ed7e31fe00190cd56b14a",
+    strip_prefix = "emsdk-cache",
+)

bazel/test_prebuilt_cache/hello-world.cc

@@ -0,0 +1,6 @@
+#include <iostream>
+
+int main(int argc, char** argv) {
+  std::cout << "hello world!" << std::endl;
+  return 0;
+}

test/test_bazel.ps1

@@ -28,3 +28,7 @@ Set-Location ..\test_secondary_lto_cache
 bazel build //:hello-world-wasm
 if (-not $?) { Exit $LastExitCode }
 
+Set-Location ..\test_prebuilt_cache
+
+bazel build //:hello-world-wasm --compilation_mode opt # test only release as used prebuilt cache is only for release builds
+if (-not $?) { Exit $LastExitCode }

test/test_bazel.sh

@@ -37,3 +37,7 @@ popd
 pushd test_secondary_lto_cache
 bazel build //:hello-world-wasm
 popd
+
+pushd test_prebuilt_cache
+bazel build //:hello-world-wasm --compilation_mode opt # test only release as used prebuilt cache is only for release builds
+popd

test/test_bazel_mac.sh

@@ -29,3 +29,5 @@ bazel build //:hello-world-wasm
 cd ../test_secondary_lto_cache
 bazel build //:hello-world-wasm
 
+cd ../test_prebuilt_cache
+bazel build //:hello-world-wasm --compilation_mode opt # test only release as used prebuilt cache is only for release builds