Skip to content

deterministic res pack generation #273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MCJEModder2026 wants to merge 1 commit into
base: dev/26.2
Choose a base branch
from
Open

deterministic res pack generation #273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions gradle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ org.gradle.jvmargs=-Xmx4G
# check these on https://fabricmc.net/use

minecraft_version=26.2
loader_version=0.19.2
loader_version=0.19.3
loom_version=1.16-SNAPSHOT

# Fabric API
fabric_api_version=0.150.2+26.2
fabric_api_version=0.155.2+26.2

maven_group = eu.pb4

Expand Down
10 changes: 7 additions & 3 deletions polymer-blocks/src/main/java/eu/pb4/polymer/blocks/api/BlockResourceCreator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package eu.pb4.polymer.blocks.api;

import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.mojang.datafixers.util.Either;
Expand Down Expand Up @@ -219,7 +220,7 @@ private void generateResources(DefaultRPBuilder builder, List<String> credits) {

}

for (var keyVal : keys) {
for (var keyVal : keys.stream().sorted(Map.Entry.comparingByKey()).toList()) {
var key = keyVal.getKey();
try {
var modelObject = new JsonObject();
Expand All @@ -237,7 +238,7 @@ private void generateResources(DefaultRPBuilder builder, List<String> credits) {


if (multipart.containsKey(key)) {
multipart.get(key).forEach(multipartObject::add);
multipart.get(key).stream().sorted(Comparator.comparing(JsonObject::toString)).forEach(multipartObject::add);

var vanillaData = builder.getDataOrSource(key);
if (vanillaData != null) {
Expand All @@ -264,7 +265,10 @@ private void generateResources(DefaultRPBuilder builder, List<String> credits) {
}

var ban2 = new JsonObject();
ban2.add("AND", bannedStates.get(key));
ban2.add("AND", bannedStates.get(key).asList().stream()
.sorted(Comparator.comparing(JsonElement::toString))
.collect(JsonArray::new, JsonArray::add, JsonArray::addAll)
);
list.add(ban2);

var when = new JsonObject();
Expand Down
2 changes: 1 addition & 1 deletion polymer-resource-pack/src/main/java/eu/pb4/polymer/resourcepack/api/ResourcePackBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ default boolean copyResourcePackFromPath(Path root, @Nullable String sourceName)
}
}

try (var str = Files.list(root)) {
try (var str = Files.list(root).sorted()) {
String finalSourceName = sourceName;
str.forEach(file -> {
try {
Expand Down
6 changes: 3 additions & 3 deletions polymer-resource-pack/src/main/java/eu/pb4/polymer/resourcepack/api/ResourcePackCreator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,15 @@ public <T> T build(OutputGenerator<T> output, ResourcePackStatusConsumer status)

var successful = true;

for (var path : this.sourcePaths) {
for (var path : this.sourcePaths.stream().sorted(Comparator.naturalOrder()).toList()) {
successful = builder.copyFromPath(path) && successful;
}

for (String modId : this.modIdsNoCopy) {
for (String modId : this.modIdsNoCopy.stream().sorted(Comparator.naturalOrder()).toList()) {
successful = builder.addAssetsSource(modId) && successful;
}

for (String modId : this.modIds) {
for (String modId : this.modIds.stream().sorted(Comparator.naturalOrder()).toList()) {
successful = builder.copyAssets(modId) && successful;
}

Expand Down
6 changes: 3 additions & 3 deletions polymer-resource-pack/src/main/java/eu/pb4/polymer/resourcepack/api/metadata/PackMcMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public static class Builder {
);
private final List<IdentifierPattern> filter = new ArrayList<>();
private final List<OverlayMetadataSection.OverlayEntry> overlay = new ArrayList<>();
private final Map<String, LanguageDefinition> language = new HashMap<>();
private final Map<String, LanguageDefinition> language = new LinkedHashMap<>();

public Builder metadata(PackMetadataSection metadata) {
this.metadata = metadata;
Expand Down Expand Up @@ -76,8 +76,8 @@ public Builder addLanguage(String name, LanguageDefinition definition) {

public PackMcMeta build() {
return new PackMcMeta(this.metadata,
this.filter.isEmpty() ? Optional.empty() : Optional.of(new ResourceFilterSection(this.filter)),
this.overlay.isEmpty() ? Optional.empty() : Optional.of(new OverlayMetadataSection(this.overlay)),
this.filter.isEmpty() ? Optional.empty() : Optional.of(new ResourceFilterSection(this.filter.stream().sorted(Comparator.comparing(Object::toString)).toList())),
this.overlay.isEmpty() ? Optional.empty() : Optional.of(new OverlayMetadataSection(this.overlay.stream().sorted(Comparator.comparing(Object::toString)).toList())),
this.language.isEmpty() ? Optional.empty() : Optional.of(new LanguageResourceMetadata(this.language))
);
}
Expand Down
4 changes: 4 additions & 0 deletions ...resource-pack/src/main/java/eu/pb4/polymer/resourcepack/impl/PolymerResourcePackImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class PolymerResourcePackImpl {
public static final String FILE_NAME;
public static final boolean IGNORE_PACK_VERSION;
public static final boolean LOG_ERRORS;
public static final boolean LOG_GENERATED_PACK_SHA1;


static {
Expand All @@ -30,6 +31,7 @@ public class PolymerResourcePackImpl {

FORCE_REQUIRE = config.markResourcePackAsRequiredByDefault;
LOG_ERRORS = config.logErrors || FabricLoader.getInstance().isDevelopmentEnvironment();
LOG_GENERATED_PACK_SHA1 = config.logGeneratedPackSha1;
}

public static Config loadConfig() {
Expand Down Expand Up @@ -61,5 +63,7 @@ public static class Config {
public String _c10 = "Toggles logging of non-critical errors when generating the pack.";
@SerializedName("log_errors")
public boolean logErrors = true;
@SerializedName("log_generated_pack_sha1")
public boolean logGeneratedPackSha1 = true;
}
}
7 changes: 4 additions & 3 deletions ...-resource-pack/src/main/java/eu/pb4/polymer/resourcepack/impl/PolymerResourcePackMod.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
import com.mojang.brigadier.context.CommandContext;
import eu.pb4.polymer.common.impl.CommonImpl;
import eu.pb4.polymer.common.impl.CommonImplUtils;
import eu.pb4.polymer.common.impl.CompatStatus;
import eu.pb4.polymer.resourcepack.api.OutputGenerator;
import eu.pb4.polymer.resourcepack.api.PolymerResourcePackUtils;
import eu.pb4.polymer.resourcepack.api.ResourcePackStatusConsumer;
import eu.pb4.polymer.resourcepack.api.metadata.PackMcMeta;
import eu.pb4.polymer.resourcepack.impl.client.rendering.PolymerResourcePack;
import net.fabricmc.api.ClientModInitializer;
import net.fabricmc.api.ModInitializer;
import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents;
import net.fabricmc.loader.api.FabricLoader;
import net.fabricmc.loader.api.metadata.CustomValue;
import net.minecraft.ChatFormatting;
Expand All @@ -36,6 +34,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import java.util.stream.StreamSupport;

import static net.minecraft.commands.Commands.literal;

Expand Down Expand Up @@ -126,7 +125,9 @@ public void onInitialize() {
continue;
}
try (var stream = Files.newDirectoryStream(folderPath, parts[parts.length - 1])){
stream.forEach(zipReader);
StreamSupport.stream(stream.spliterator(), false)
.sorted()
.forEach(zipReader);
}
} else {
zipReader.accept(gamePath.resolve(field));
Expand Down
34 changes: 26 additions & 8 deletions ...urce-pack/src/main/java/eu/pb4/polymer/resourcepack/impl/generation/DefaultRPBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileTime;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiConsumer;
Expand Down Expand Up @@ -193,7 +194,7 @@ public boolean copyFromPath(Path basePath, String targetPrefix, boolean override
if (Files.isDirectory(basePath)) {
status.accept("action:copy_path_start/" + sourceName);
Path finalBasePath = basePath;
try (var str = Files.walk(basePath)) {
try (var str = Files.walk(basePath).sorted()) {
str.forEach((file) -> {
var relative = finalBasePath.relativize(file);
var path = targetPrefix + relative.toString().replace("\\", "/");
Expand Down Expand Up @@ -246,7 +247,7 @@ public boolean copyAssets(String modId) {
this.modsList.add(container);
try {
for (var rootPaths : container.getRootPaths()) {
try (var str = Files.list(rootPaths)) {
try (var str = Files.list(rootPaths).sorted()) {
str.forEach(file -> {
try {
var name = file.getFileName().toString();
Expand All @@ -272,7 +273,7 @@ public boolean copyAssets(String modId) {
for (var x : baseToCopy) {
Path assets = rootPaths.resolve(x);
if (Files.exists(assets)) {
try (var str = Files.walk(assets)) {
try (var str = Files.walk(assets).sorted()) {
str.forEach((file) -> {
var relative = assets.relativize(file);
var path = relative.toString().replace("\\", "/");
Expand Down Expand Up @@ -324,7 +325,9 @@ public byte[] getDataOrSource(String path) {

@Override
public void forEachResource(BiConsumer<String, PackResource> consumer) {
Map.copyOf(this.fileMap).forEach(consumer);
var sorted = new ArrayList<>(this.fileMap.entrySet());
sorted.sort(Map.Entry.comparingByKey());
sorted.forEach(e -> consumer.accept(e.getKey(), e.getValue()));
}

@Override
Expand Down Expand Up @@ -438,14 +441,24 @@ public CompletableFuture<T> buildResourcePack() {

status.accept("action:merge_files_start");

for (var entry : this.atlasDefinitions.entrySet()) {
var sortedAtlas = new ArrayList<>(this.atlasDefinitions.entrySet());
sortedAtlas.sort(Map.Entry.comparingByKey());
for (var entry : sortedAtlas) {
var obj = new JsonObject();
obj.add("sources", entry.getValue());
this.fileMap.put(entry.getKey(), PackResource.of(obj.toString().getBytes(StandardCharsets.UTF_8)));
}

for (var entry : this.objectMergeDefinitions.entrySet()) {
this.fileMap.put(entry.getKey(), PackResource.of(entry.getValue().toString().getBytes(StandardCharsets.UTF_8)));
var sortedObjects = new ArrayList<>(this.objectMergeDefinitions.entrySet());
sortedObjects.sort(Map.Entry.comparingByKey());
for (var entry : sortedObjects) {
var sortedJson = new JsonObject();
var keys = new ArrayList<>(entry.getValue().keySet());
keys.sort(String::compareTo);
for (var key : keys) {
sortedJson.add(key, entry.getValue().get(key));
}
this.fileMap.put(entry.getKey(), PackResource.of(sortedJson.toString().getBytes(StandardCharsets.UTF_8)));
}

this.fileMap.put(AssetPaths.PACK_METADATA, PackResource.of(this.packMetadata.build().asString().getBytes(StandardCharsets.UTF_8)));
Expand Down Expand Up @@ -519,7 +532,7 @@ public static OutputGenerator.Result writeSingleZip(Path out, Collection<Map.Ent
try (var outputStream = new ZipOutputStream(new BufferedOutputStream(
new HashingStream(Files.newOutputStream(out, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING), Hashing.sha1(), hashCode::setValue), 65536)
)) {
for (var entry : resources) {
for (var entry : resources.stream().sorted(Map.Entry.comparingByKey()).toList()) {
var path = entry.getKey();
var resource = entry.getValue();

Expand All @@ -532,6 +545,9 @@ public static OutputGenerator.Result writeSingleZip(Path out, Collection<Map.Ent

var zipEntry = new ZipEntry(path);
zipEntry.setTime(0);
zipEntry.setLastAccessTime(FileTime.fromMillis(0));
zipEntry.setLastModifiedTime(FileTime.fromMillis(0));
zipEntry.setCreationTime(FileTime.fromMillis(0));
outputStream.putNextEntry(zipEntry);
if (resource != null) {
resource.getStream().transferTo(outputStream);
Expand All @@ -543,6 +559,8 @@ public static OutputGenerator.Result writeSingleZip(Path out, Collection<Map.Ent
LOGGER.warn("Failed to write the zip file!", e);
return null;
}
if (PolymerResourcePackImpl.LOG_GENERATED_PACK_SHA1)
LOGGER.info("Generated pack SHA1: {}", hashCode);
status.accept("action:write_zip_end");
return new OutputGenerator.Result(out, hashCode.toString(), status.hadIssues());
}
Expand Down
2 changes: 1 addition & 1 deletion polymer-sound-patcher/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ dependencies {
implementation(project(path: ':polymer-common'))
implementation(project(path: ':polymer-resource-pack'))
implementation(project(path: ':polymer-resource-pack-extras'))
compileOnly(project(path: ':polymer-core'))
implementation(project(path: ':polymer-core'))

testmodImplementation sourceSets.main.output
}
Expand Down
Loading