Rewrite Checksum util class (#1304)

* Rewrite Checksum util class as the old one was annoying me.

* Small cleanup in UnpickService

* Test fixes & cleanup

* Fix build

* Fix OfflineModeTest
This commit is contained in:
modmuss
2025-05-07 12:13:35 +01:00
committed by GitHub
parent eff00a1c30
commit 0a35910c63
22 changed files with 150 additions and 152 deletions

View File

@@ -37,7 +37,6 @@ import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.hash.Hashing;
import com.google.gson.JsonObject;
import org.apache.commons.io.FileUtils;
import org.gradle.api.artifacts.ArtifactView;
@@ -61,6 +60,7 @@ import org.slf4j.LoggerFactory;
import net.fabricmc.loom.LoomGradlePlugin;
import net.fabricmc.loom.task.AbstractLoomTask;
import net.fabricmc.loom.util.Checksum;
import net.fabricmc.loom.util.ZipReprocessorUtil;
import net.fabricmc.loom.util.fmj.FabricModJsonFactory;
@@ -170,9 +170,7 @@ public abstract class NestableJarGenerationTask extends AbstractLoomTask {
// Fabric Loader can't handle modIds longer than 64 characters
if (modId.length() > 64) {
String hash = Hashing.sha256()
.hashString(modId, StandardCharsets.UTF_8)
.toString();
String hash = Checksum.of(modId).sha256().hex();
modId = modId.substring(0, 50) + hash.substring(0, 14);
}

View File

@@ -275,7 +275,7 @@ public abstract class CompileConfiguration implements Runnable {
private LockFile getLockFile() {
final LoomGradleExtension extension = LoomGradleExtension.get(getProject());
final Path cacheDirectory = extension.getFiles().getUserCache().toPath();
final String pathHash = Checksum.projectHash(getProject());
final String pathHash = Checksum.of(getProject()).sha1().hex();
return new LockFile(
cacheDirectory.resolve("." + pathHash + ".lock"),
"Lock for cache='%s', project='%s'".formatted(

View File

@@ -25,7 +25,6 @@
package net.fabricmc.loom.configuration.accesswidener;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -40,11 +39,7 @@ import net.fabricmc.tinyremapper.TinyRemapper;
public record LocalAccessWidenerEntry(Path path, String hash) implements AccessWidenerEntry {
public static LocalAccessWidenerEntry create(Path path) {
try {
return new LocalAccessWidenerEntry(path, Checksum.sha1Hex(path));
} catch (IOException e) {
throw new UncheckedIOException("Failed to create LocalAccessWidenerEntry", e);
}
return new LocalAccessWidenerEntry(path, Checksum.of(path).sha1().hex());
}
@Override

View File

@@ -69,7 +69,7 @@ public interface ArtifactRef {
}
public String version() {
return replaceIfNullOrEmpty(artifact.getModuleVersion().getId().getVersion(), () -> Checksum.truncatedSha256(artifact.getFile()));
return replaceIfNullOrEmpty(artifact.getModuleVersion().getId().getVersion(), () -> Checksum.of(artifact.getFile()).sha256().hex(10));
}
public String classifier() {

View File

@@ -275,7 +275,7 @@ public class ModConfigurationRemapper {
for (File artifact : files) {
final String name = getNameWithoutExtension(artifact.toPath());
final String version = replaceIfNullOrEmpty(dependency.getVersion(), () -> Checksum.truncatedSha256(artifact));
final String version = replaceIfNullOrEmpty(dependency.getVersion(), () -> Checksum.of(artifact).sha256().hex(10));
artifacts.add(new ArtifactRef.FileArtifactRef(artifact.toPath(), group, name, version));
}
}

View File

@@ -25,7 +25,6 @@
package net.fabricmc.loom.configuration.processors;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -113,11 +112,11 @@ public final class MinecraftJarProcessorManager {
public String getJarHash() {
//fabric-loom:mod-javadoc:-1289977000
return Checksum.sha1Hex(getCacheValue().getBytes(StandardCharsets.UTF_8)).substring(0, 10);
return Checksum.of(getCacheValue()).sha1().hex(10);
}
public String getSourceMappingsHash() {
return Checksum.sha1Hex(getCacheValue().getBytes(StandardCharsets.UTF_8));
return Checksum.of(getCacheValue()).sha1().hex();
}
public boolean requiresProcessingJar(Path jar) {

View File

@@ -124,7 +124,7 @@ public abstract class ModJavadocProcessor implements MinecraftJarProcessor<ModJa
try {
final byte[] data = fabricModJson.getSource().read(javaDocPath);
mappingsHash = Checksum.sha1Hex(data);
mappingsHash = Checksum.of(data).sha1().hex();
try (Reader reader = new InputStreamReader(new ByteArrayInputStream(data))) {
MappingReader.read(reader, mappings);

View File

@@ -25,7 +25,6 @@
package net.fabricmc.loom.configuration.providers.mappings;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
@@ -115,7 +114,7 @@ public abstract class IntermediaryMappingsProvider extends IntermediateMappingsP
if (!LoomGradleExtensionApiImpl.DEFAULT_INTERMEDIARY_URL.equals(urlRaw)) {
final String url = getIntermediaryUrl().get().formatted(encodedMcVersion);
return NAME + "-" + Checksum.sha1Hex(url.getBytes(StandardCharsets.UTF_8));
return NAME + "-" + Checksum.of(url).sha1().hex();
}
return NAME;

View File

@@ -29,8 +29,8 @@ import java.nio.file.Path;
import java.util.Arrays;
import java.util.Objects;
import net.fabricmc.loom.api.mappings.layered.spec.FileSpec;
import net.fabricmc.loom.api.mappings.layered.MappingContext;
import net.fabricmc.loom.api.mappings.layered.spec.FileSpec;
import net.fabricmc.loom.util.Checksum;
public class LocalFileSpec implements FileSpec {
@@ -48,7 +48,7 @@ public class LocalFileSpec implements FileSpec {
}
// Use the file hash as part of the spec, this means if the input file changes the mappings will be re-generated.
return Objects.hash(Arrays.hashCode(Checksum.sha256(file)), file.getAbsolutePath());
return Objects.hash(Arrays.hashCode(Checksum.of(file).sha256().digest()), file.getAbsolutePath());
}
@Override

View File

@@ -25,7 +25,6 @@
package net.fabricmc.loom.configuration.providers.minecraft.verify;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import java.util.function.Function;
@@ -86,9 +85,9 @@ public abstract class MinecraftJarVerification {
}
LOGGER.info("Found executed hash ({}) for known version: {}", expectedHash, version);
String hash = Checksum.sha256Hex(Files.readAllBytes(path));
Checksum.Result hash = Checksum.of(path).sha256();
if (hash.equalsIgnoreCase(expectedHash)) {
if (hash.matchesStr(expectedHash)) {
LOGGER.info("Minecraft {} hash matches known version", path.getFileName());
return true;
}

View File

@@ -105,13 +105,13 @@ public record ClassEntry(String name, List<String> innerClasses, List<String> su
public String hash(Path root) throws IOException {
StringJoiner joiner = new StringJoiner(",");
joiner.add(Checksum.sha256Hex(Files.readAllBytes(root.resolve(name))));
joiner.add(Checksum.of(root.resolve(name)).sha256().hex());
for (String innerClass : innerClasses) {
joiner.add(Checksum.sha256Hex(Files.readAllBytes(root.resolve(innerClass))));
joiner.add(Checksum.of(root.resolve(innerClass)).sha256().hex());
}
return Checksum.sha256Hex(joiner.toString().getBytes());
return Checksum.of(joiner.toString()).sha256().hex();
}
/**
@@ -138,7 +138,7 @@ public record ClassEntry(String name, List<String> innerClasses, List<String> su
}
}
return Checksum.sha256Hex(joiner.toString().getBytes());
return Checksum.of(joiner.toString()).sha256().hex();
}
public String sourcesFileName() {

View File

@@ -409,17 +409,13 @@ public abstract class GenerateSourcesTask extends AbstractLoomTask {
getLogger().info("Decompile cache data: {}", sj);
try {
return Checksum.sha256Hex(sj.toString().getBytes(StandardCharsets.UTF_8));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return Checksum.of(sj.toString()).sha256().hex();
}
private String getDecompilerCheckKey() {
var sj = new StringJoiner(",");
sj.add(decompilerOptions.getDecompilerClassName().get());
sj.add(Checksum.fileCollectionHash(decompilerOptions.getClasspath()));
sj.add(Checksum.of(decompilerOptions.getClasspath()).sha256().hex());
for (Map.Entry<String, String> entry : decompilerOptions.getOptions().get().entrySet()) {
sj.add(entry.getKey() + "=" + entry.getValue());

View File

@@ -29,7 +29,6 @@ import java.lang.reflect.InvocationTargetException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.StringJoiner;
import org.gradle.api.Project;
import org.gradle.api.artifacts.ConfigurationContainer;
@@ -149,16 +148,11 @@ public class UnpickService extends Service<UnpickService.Options> {
}
public String getUnpickCacheKey() {
if (!getOptions().getUnpickDefinitions().isPresent()) {
return "";
}
var sj = new StringJoiner(",");
sj.add(Checksum.fileHash(getOptions().getUnpickDefinitions().getAsFile().get()));
sj.add(Checksum.fileCollectionHash(getOptions().getUnpickConstantJar()));
sj.add(Checksum.fileCollectionHash(getOptions().getUnpickRuntimeClasspath()));
return sj.toString();
return Checksum.of(List.of(
Checksum.of(getOptions().getUnpickDefinitions().getAsFile().get()),
Checksum.of(getOptions().getUnpickConstantJar()),
Checksum.of(getOptions().getUnpickRuntimeClasspath())
)).sha256().hex();
}
public interface UnpickParams extends WorkParameters {

View File

@@ -24,7 +24,6 @@
package net.fabricmc.loom.util;
import java.nio.charset.StandardCharsets;
import java.util.function.Supplier;
import com.google.common.base.Suppliers;
@@ -40,7 +39,7 @@ import net.fabricmc.loom.util.gradle.GradleTypeAdapter;
public abstract class CacheKey {
private static final int CHECKSUM_LENGTH = 8;
private final transient Supplier<String> jsonSupplier = Suppliers.memoize(() -> GradleTypeAdapter.GSON.toJson(this));
private final transient Supplier<String> cacheKeySupplier = Suppliers.memoize(() -> Checksum.sha1Hex(jsonSupplier.get().getBytes(StandardCharsets.UTF_8)).substring(0, CHECKSUM_LENGTH));
private final transient Supplier<String> cacheKeySupplier = Suppliers.memoize(() -> Checksum.of(jsonSupplier.get()).sha1().hex(CHECKSUM_LENGTH));
public static <T> T create(Project project, Class<T> clazz, Action<T> action) {
T instance = project.getObjects().newInstance(clazz);

View File

@@ -1,7 +1,7 @@
/*
* This file is part of fabric-loom, licensed under the MIT License (MIT).
*
* Copyright (c) 2016-2020 FabricMC
* Copyright (c) 2025 FabricMC
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -26,105 +26,138 @@ package net.fabricmc.loom.util;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.StringJoiner;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
import java.util.List;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
import org.gradle.api.Project;
import org.gradle.api.file.FileCollection;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.jetbrains.annotations.NotNull;
public class Checksum {
private static final Logger log = Logging.getLogger(Checksum.class);
public final class Checksum {
public static Checksum of(byte[] data) {
return new Checksum(digest -> digest.write(data));
}
public static boolean equals(File file, String checksum) {
if (file == null || !file.exists()) {
return false;
}
public static Checksum of(String str) {
return new Checksum(digest -> digest.write(str));
}
public static Checksum of(File file) {
return of(file.toPath());
}
public static Checksum of(Path file) {
return new Checksum(digest -> {
try (InputStream is = Files.newInputStream(file)) {
is.transferTo(digest);
}
});
}
public static Checksum of(Project project) {
return of(project.getProjectDir().getAbsolutePath() + ":" + project.getPath());
}
public static Checksum of(FileCollection files) {
return new Checksum(os -> {
for (File file : files) {
try (InputStream is = Files.newInputStream(file.toPath())) {
is.transferTo(os);
}
}
});
}
public static Checksum of(List<Checksum> others) {
return new Checksum(os -> {
for (Checksum other : others) {
other.consumer.accept(os);
}
});
}
private final DataConsumer consumer;
private Checksum(DataConsumer consumer) {
this.consumer = consumer;
}
public Result sha1() {
return computeResult("SHA-1");
}
public Result sha256() {
return computeResult("SHA-256");
}
public Result md5() {
return computeResult("MD5");
}
private Result computeResult(String algorithm) {
MessageDigest digest;
try {
HashCode hash = Files.asByteSource(file).hash(Hashing.sha1());
log.debug("Checksum check: '" + hash.toString() + "' == '" + checksum + "'?");
return hash.toString().equals(checksum);
} catch (IOException e) {
e.printStackTrace();
digest = MessageDigest.getInstance(algorithm);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
return false;
try (MessageDigestOutputStream os = new MessageDigestOutputStream(digest)) {
consumer.accept(os);
} catch (IOException e) {
throw new UncheckedIOException("Failed to compute checksum", e);
}
return new Result(digest.digest());
}
public static byte[] sha256(File file) {
try {
HashCode hash = Files.asByteSource(file).hash(Hashing.sha256());
return hash.asBytes();
} catch (IOException e) {
throw new UncheckedIOException("Failed to get file hash", e);
public record Result(byte[] digest) {
public String hex() {
return HexFormat.of().formatHex(digest());
}
public String hex(int length) {
return hex().substring(0, length);
}
public boolean matchesStr(String other) {
return hex().equalsIgnoreCase(other);
}
}
public static String sha256Hex(byte[] input) throws IOException {
HashCode hash = ByteSource.wrap(input).hash(Hashing.sha256());
return Checksum.toHex(hash.asBytes());
@FunctionalInterface
private interface DataConsumer {
void accept(MessageDigestOutputStream os) throws IOException;
}
public static String sha1Hex(Path path) throws IOException {
HashCode hash = Files.asByteSource(path.toFile()).hash(Hashing.sha1());
return toHex(hash.asBytes());
}
private static class MessageDigestOutputStream extends OutputStream {
private final MessageDigest digest;
public static String sha1Hex(byte[] input) {
try {
HashCode hash = ByteSource.wrap(input).hash(Hashing.sha1());
return toHex(hash.asBytes());
} catch (IOException e) {
throw new UncheckedIOException("Failed to hash", e);
private MessageDigestOutputStream(MessageDigest digest) {
this.digest = digest;
}
}
public static String truncatedSha256(File file) {
try {
HashCode hash = Files.asByteSource(file).hash(Hashing.sha256());
return hash.toString().substring(0, 12);
} catch (IOException e) {
throw new UncheckedIOException("Failed to get file hash of " + file, e);
@Override
public void write(int b) {
digest.update((byte) b);
}
}
public static String toHex(byte[] bytes) {
return BaseEncoding.base16().lowerCase().encode(bytes);
}
public static String projectHash(Project project) {
String str = project.getProjectDir().getAbsolutePath() + ":" + project.getPath();
String hex = sha1Hex(str.getBytes(StandardCharsets.UTF_8));
return hex.substring(hex.length() - 16);
}
public static String fileHash(File file) {
try {
return Checksum.sha256Hex(java.nio.file.Files.readAllBytes(file.toPath()));
} catch (IOException e) {
throw new UncheckedIOException(e);
@Override
public void write(byte @NotNull[] b, int off, int len) {
digest.update(b, off, len);
}
}
public static String fileCollectionHash(FileCollection files) {
var sj = new StringJoiner(",");
files.getFiles()
.stream()
.sorted(Comparator.comparing(File::getAbsolutePath))
.map(Checksum::fileHash)
.forEach(sj::add);
return sj.toString();
public void write(String string) throws IOException {
write(string.getBytes(StandardCharsets.UTF_8));
}
}
}

View File

@@ -222,7 +222,7 @@ public final class Download {
String downloadedHash;
try {
downloadedHash = Checksum.sha1Hex(output);
downloadedHash = Checksum.of(output).sha1().hex();
Files.deleteIfExists(output);
} catch (IOException e) {
downloadedHash = "unknown hash";
@@ -357,12 +357,12 @@ public final class Download {
String hash = expectedHash.substring(i + 1);
try {
String computedHash = switch (algorithm) {
case "sha1" -> Checksum.sha1Hex(path);
Checksum.Result computedHash = switch (algorithm) {
case "sha1" -> Checksum.of(path).sha1();
default -> throw error("Unsupported hash algorithm (%s)", algorithm);
};
return computedHash.equalsIgnoreCase(hash);
return computedHash.matchesStr(hash);
} catch (IOException e) {
throw new UncheckedIOException(e);
}

View File

@@ -47,7 +47,7 @@ class OfflineModeTest extends Specification implements GradleProjectTestTrait {
}
import net.fabricmc.loom.util.Checksum
def projectHash = Checksum.projectHash(getProject())
def projectHash = Checksum.of(getProject()).sha1().hex()
println("%%" + projectHash + "%%")
""".stripIndent()

View File

@@ -24,17 +24,16 @@
package net.fabricmc.loom.test.integration
import com.google.common.hash.HashCode
import com.google.common.hash.Hashing
import com.google.common.io.Files
import spock.lang.Specification
import spock.lang.Unroll
import spock.util.environment.RestoreSystemProperties
import net.fabricmc.loom.test.util.GradleProjectTestTrait
import net.fabricmc.loom.util.Checksum
import static java.lang.System.setProperty
import static net.fabricmc.loom.test.LoomTestConstants.*
import static net.fabricmc.loom.test.LoomTestConstants.DEFAULT_GRADLE
import static net.fabricmc.loom.test.LoomTestConstants.PRE_RELEASE_GRADLE
import static org.gradle.testkit.runner.TaskOutcome.SUCCESS
class ReproducibleBuildTest extends Specification implements GradleProjectTestTrait {
@@ -60,7 +59,6 @@ class ReproducibleBuildTest extends Specification implements GradleProjectTestTr
}
String generateMD5(File file) {
HashCode hash = Files.asByteSource(file).hash(Hashing.md5())
return hash.asBytes().encodeHex() as String
return Checksum.of(file).md5().hex()
}
}

View File

@@ -37,7 +37,7 @@ class ChecksumTest extends Specification {
project.getProjectDir() >> new File(dir)
when:
def hash = Checksum.projectHash(project)
def hash = Checksum.of(project).sha256().hex()
then:
!hash.empty

View File

@@ -180,7 +180,7 @@ class ZipUtilsTest extends Specification {
then:
ZipUtils.unpack(zip, "text.txt") == "hello world".bytes
ZipUtils.unpack(zip, "fabric.mod.json") == "Some text".bytes
Checksum.sha1Hex(zip) == "1b06cc0aaa65ab2b0d423fe33431ff5bd14bf9c8"
Checksum.of(zip).sha1().hex() == "1b06cc0aaa65ab2b0d423fe33431ff5bd14bf9c8"
where:
timezone | _
@@ -256,6 +256,6 @@ class ZipUtilsTest extends Specification {
then:
ZipUtils.unpack(zip, "text.txt") == "hello world".bytes
Checksum.sha1Hex(zip) == "e699fa52a520553241aac798f72255ac0a912b05"
Checksum.of(zip).sha1().hex() == "e699fa52a520553241aac798f72255ac0a912b05"
}
}

View File

@@ -452,6 +452,6 @@ class DownloadFileTest extends DownloadTest {
.downloadPath(file)
then:
Checksum.sha1Hex(file) == "8e8c9be5dc27802caba47053d4fdea328f7f89bd"
Checksum.of(file).sha1().hex() == "8e8c9be5dc27802caba47053d4fdea328f7f89bd"
}
}

View File

@@ -25,7 +25,6 @@
package net.fabricmc.loom.test.util
import java.nio.file.Path
import java.security.MessageDigest
import net.fabricmc.loom.LoomGradlePlugin
import net.fabricmc.loom.configuration.providers.BundleMetadata
@@ -35,6 +34,7 @@ import net.fabricmc.loom.configuration.providers.minecraft.verify.CertificateCha
import net.fabricmc.loom.configuration.providers.minecraft.verify.JarVerifier
import net.fabricmc.loom.configuration.providers.minecraft.verify.KnownVersions
import net.fabricmc.loom.configuration.providers.minecraft.verify.SignatureVerificationFailure
import net.fabricmc.loom.util.Checksum
import net.fabricmc.loom.util.Constants
import net.fabricmc.loom.util.download.Download
import net.fabricmc.loom.util.download.DownloadExecutor
@@ -109,13 +109,13 @@ class KnownVersionsGenerator {
def clientJar = dir.resolve(client.sha1() + ".jar")
if (!isSigned(clientJar)) {
unsignedClientVersions.put(version.id, sha256(clientJar))
unsignedClientVersions.put(version.id, Checksum.of(clientJar).sha256().hex())
}
if (server != null) {
def serverJar = dir.resolve(server.sha1() + ".jar")
if (BundleMetadata.fromJar(serverJar) == null) {
unsignedServerVersions.put(version.id, sha256(serverJar))
unsignedServerVersions.put(version.id, Checksum.of(serverJar).sha256().hex())
}
}
}
@@ -128,16 +128,4 @@ class KnownVersionsGenerator {
return false
}
}
static String sha256(Path path) {
MessageDigest md = MessageDigest.getInstance("SHA-256")
path.withInputStream { inputStream ->
byte[] buffer = new byte[8192]
int bytesRead
while ((bytesRead = inputStream.read(buffer)) != -1) {
md.update(buffer, 0, bytesRead)
}
}
return md.digest().encodeHex().toString()
}
}