From 4b5217649949f141e2c179e3bc89e82ab9ef7779 Mon Sep 17 00:00:00 2001 From: modmuss50 Date: Fri, 28 Jan 2022 20:26:16 +0000 Subject: [PATCH] Cleanup asset downloading, add support for legacy and pre-1.6 assets (#585) --- .../minecraft/assets/AssetIndex.java | 32 +++- .../minecraft/assets/AssetObject.java | 29 --- .../assets/MinecraftAssetsProvider.java | 162 ----------------- .../loom/task/DownloadAssetsTask.java | 169 +++++++++++++++++- .../task/launch/GenerateDLIConfigTask.java | 10 +- .../loom/util/HashedDownloadUtil.java | 8 - 6 files changed, 199 insertions(+), 211 deletions(-) delete mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetObject.java delete mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/MinecraftAssetsProvider.java diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetIndex.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetIndex.java index c87fc8c8..3dd6a3c5 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetIndex.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetIndex.java @@ -24,18 +24,38 @@ package net.fabricmc.loom.configuration.providers.minecraft.assets; -import java.util.HashSet; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonProperty; @SuppressWarnings("unused") -public record AssetIndex(Map objects, boolean virtual) { +public record AssetIndex(Map objects, boolean virtual, @JsonProperty("map_to_resources") boolean mapToResources) { public AssetIndex() { - this(new LinkedHashMap<>(), false); + this(new LinkedHashMap<>(), false, false); } - public Set getUniqueObjects() { - return new HashSet<>(this.objects.values()); + public Collection getObjects() { + return objects.entrySet().stream().map(Object::new).toList(); + } + + public record Entry(String hash, long size) { + } + + public record Object(String path, String hash, long size) { + private Object(Map.Entry entry) { + this(entry.getKey(), entry.getValue().hash(), entry.getValue().size()); + } + + public String name() { + int end = path().lastIndexOf("/") + 1; + + if (end > 0) { + return path().substring(end); + } + + return path(); + } } } diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetObject.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetObject.java deleted file mode 100644 index b7e63b12..00000000 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetObject.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * This file is part of fabric-loom, licensed under the MIT License (MIT). - * - * Copyright (c) 2016-2021 FabricMC - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package net.fabricmc.loom.configuration.providers.minecraft.assets; - -@SuppressWarnings("unused") -public record AssetObject(String hash, long size) { -} diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/MinecraftAssetsProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/MinecraftAssetsProvider.java deleted file mode 100644 index 175e85ed..00000000 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/MinecraftAssetsProvider.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * This file is part of fabric-loom, licensed under the MIT License (MIT). - * - * Copyright (c) 2018-2021 FabricMC - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package net.fabricmc.loom.configuration.providers.minecraft.assets; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.URL; -import java.util.Deque; -import java.util.Map; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import com.google.common.base.Stopwatch; -import org.gradle.api.GradleException; -import org.gradle.api.Project; - -import net.fabricmc.loom.LoomGradleExtension; -import net.fabricmc.loom.LoomGradlePlugin; -import net.fabricmc.loom.configuration.providers.minecraft.MinecraftProvider; -import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionMeta; -import net.fabricmc.loom.util.MirrorUtil; -import net.fabricmc.loom.util.HashedDownloadUtil; -import net.fabricmc.loom.util.gradle.ProgressLoggerHelper; - -public class MinecraftAssetsProvider { - public static void provide(MinecraftProvider minecraftProvider, Project project) throws IOException { - LoomGradleExtension extension = LoomGradleExtension.get(project); - boolean offline = project.getGradle().getStartParameter().isOffline(); - - MinecraftVersionMeta versionInfo = minecraftProvider.getVersionInfo(); - MinecraftVersionMeta.AssetIndex assetIndex = versionInfo.assetIndex(); - - // get existing cache files - File assets = new File(extension.getFiles().getUserCache(), "assets"); - - if (!assets.exists()) { - assets.mkdirs(); - } - - File assetsInfo = new File(assets, "indexes" + File.separator + assetIndex.fabricId(minecraftProvider.minecraftVersion()) + ".json"); - - project.getLogger().info(":downloading asset index"); - - if (offline) { - if (assetsInfo.exists()) { - //We know it's outdated but can't do anything about it, oh well - project.getLogger().warn("Asset index outdated"); - } else { - //We don't know what assets we need, just that we don't have any - throw new GradleException("Asset index not found at " + assetsInfo.getAbsolutePath()); - } - } else { - HashedDownloadUtil.downloadIfInvalid(new URL(assetIndex.url()), assetsInfo, assetIndex.sha1(), project.getLogger(), false); - } - - Deque loggers = new ConcurrentLinkedDeque<>(); - ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1))); - - AssetIndex index; - - try (FileReader fileReader = new FileReader(assetsInfo)) { - index = LoomGradlePlugin.OBJECT_MAPPER.readValue(fileReader, AssetIndex.class); - } - - Stopwatch stopwatch = Stopwatch.createStarted(); - - Map parent = index.objects(); - - for (Map.Entry entry : parent.entrySet()) { - AssetObject object = entry.getValue(); - String sha1 = object.hash(); - String filename = "objects" + File.separator + sha1.substring(0, 2) + File.separator + sha1; - File file = new File(assets, filename); - - if (offline) { - if (file.exists()) { - project.getLogger().warn("Outdated asset " + entry.getKey()); - } else { - throw new GradleException("Asset " + entry.getKey() + " not found at " + file.getAbsolutePath()); - } - } else { - executor.execute(() -> { - final String[] assetName = {entry.getKey()}; - int end = assetName[0].lastIndexOf("/") + 1; - - if (end > 0) { - assetName[0] = assetName[0].substring(end); - } - - project.getLogger().debug("validating asset " + assetName[0]); - - final ProgressLoggerHelper[] progressLogger = new ProgressLoggerHelper[1]; - - try { - HashedDownloadUtil.downloadIfInvalid(new URL(MirrorUtil.getResourcesBase(project) + sha1.substring(0, 2) + "/" + sha1), file, sha1, project.getLogger(), true, () -> { - ProgressLoggerHelper logger = loggers.pollFirst(); - - if (logger == null) { - //Create a new logger if we need one - progressLogger[0] = ProgressLoggerHelper.getProgressFactory(project, MinecraftAssetsProvider.class.getName()); - progressLogger[0].start("Downloading assets...", "assets"); - } else { - // use a free logger if we can - progressLogger[0] = logger; - } - - project.getLogger().debug("downloading asset " + assetName[0]); - progressLogger[0].progress(String.format("%-30.30s", assetName[0]) + " - " + sha1); - }); - } catch (IOException e) { - throw new RuntimeException("Failed to download: " + assetName[0], e); - } - - if (progressLogger[0] != null) { - //Give this logger back if we used it - loggers.add(progressLogger[0]); - } - }); - } - } - - project.getLogger().info("Took " + stopwatch.stop() + " to iterate " + parent.size() + " asset index."); - - //Wait for the assets to all download - executor.shutdown(); - - try { - if (executor.awaitTermination(2, TimeUnit.HOURS)) { - executor.shutdownNow(); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - loggers.forEach(ProgressLoggerHelper::completed); - } -} diff --git a/src/main/java/net/fabricmc/loom/task/DownloadAssetsTask.java b/src/main/java/net/fabricmc/loom/task/DownloadAssetsTask.java index f03a1079..c4c204b3 100644 --- a/src/main/java/net/fabricmc/loom/task/DownloadAssetsTask.java +++ b/src/main/java/net/fabricmc/loom/task/DownloadAssetsTask.java @@ -24,20 +24,179 @@ package net.fabricmc.loom.task; +import java.io.File; +import java.io.FileReader; import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URL; +import java.util.Deque; +import java.util.Objects; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import javax.inject.Inject; + +import org.gradle.api.GradleException; import org.gradle.api.Project; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.OutputDirectory; import org.gradle.api.tasks.TaskAction; import net.fabricmc.loom.LoomGradleExtension; -import net.fabricmc.loom.configuration.providers.minecraft.assets.MinecraftAssetsProvider; +import net.fabricmc.loom.LoomGradlePlugin; +import net.fabricmc.loom.configuration.ide.RunConfigSettings; +import net.fabricmc.loom.configuration.providers.minecraft.MinecraftProvider; +import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionMeta; +import net.fabricmc.loom.configuration.providers.minecraft.assets.AssetIndex; +import net.fabricmc.loom.util.HashedDownloadUtil; +import net.fabricmc.loom.util.MirrorUtil; +import net.fabricmc.loom.util.gradle.ProgressLoggerHelper; + +public abstract class DownloadAssetsTask extends AbstractLoomTask { + @Input + public abstract Property getAssetsHash(); + + @OutputDirectory + public abstract RegularFileProperty getAssetsDirectory(); + + @OutputDirectory + public abstract RegularFileProperty getLegacyResourcesDirectory(); + + @Inject + public DownloadAssetsTask() { + final MinecraftVersionMeta versionInfo = getExtension().getMinecraftProvider().getVersionInfo(); + final File assetsDir = new File(getExtension().getFiles().getUserCache(), "assets"); + + getAssetsDirectory().set(assetsDir); + getAssetsHash().set(versionInfo.assetIndex().sha1()); + + if (versionInfo.assets().equals("legacy")) { + getLegacyResourcesDirectory().set(new File(assetsDir, "/legacy/" + versionInfo.id())); + } else { + // pre-1.6 resources + RunConfigSettings client = Objects.requireNonNull(getExtension().getRunConfigs().findByName("client"), "Could not find client run config"); + getLegacyResourcesDirectory().set(new File(getProject().getProjectDir(), client.getRunDir() + "/resources")); + } + + getAssetsHash().finalizeValueOnRead(); + getAssetsDirectory().finalizeValueOnRead(); + getLegacyResourcesDirectory().finalizeValueOnRead(); + } -public class DownloadAssetsTask extends AbstractLoomTask { @TaskAction public void downloadAssets() throws IOException { - Project project = this.getProject(); - LoomGradleExtension extension = getExtension(); + final Project project = this.getProject(); + final File assetsDirectory = getAssetsDirectory().get().getAsFile(); + final Deque loggers = new ConcurrentLinkedDeque<>(); + final ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1))); + final AssetIndex assetIndex = getAssetIndex(); - MinecraftAssetsProvider.provide(extension.getMinecraftProvider(), project); + if (!assetsDirectory.exists()) { + assetsDirectory.mkdirs(); + } + + if (assetIndex.mapToResources()) { + getLegacyResourcesDirectory().get().getAsFile().mkdirs(); + } + + for (AssetIndex.Object object : assetIndex.getObjects()) { + final String path = object.path(); + final String sha1 = object.hash(); + final File file = getAssetsFile(object, assetIndex); + + if (getProject().getGradle().getStartParameter().isOffline()) { + if (!file.exists()) { + throw new GradleException("Asset " + path + " not found at " + file.getAbsolutePath()); + } + + continue; + } + + final Supplier getOrCreateLogger = () -> { + ProgressLoggerHelper logger = loggers.pollFirst(); + + if (logger == null) { + // No logger available, create a new one + logger = ProgressLoggerHelper.getProgressFactory(project, DownloadAssetsTask.class.getName()); + logger.start("Downloading assets...", "assets"); + } + + return logger; + }; + + executor.execute(() -> { + final ProgressLoggerHelper logger = getOrCreateLogger.get(); + + try { + HashedDownloadUtil.downloadIfInvalid(new URL(MirrorUtil.getResourcesBase(project) + sha1.substring(0, 2) + "/" + sha1), file, sha1, project.getLogger(), true, () -> { + project.getLogger().debug("downloading asset " + object.name()); + logger.progress(String.format("%-30.30s", object.name()) + " - " + sha1); + }); + } catch (IOException e) { + throw new UncheckedIOException("Failed to download: " + object.name(), e); + } + + // Give this logger back + loggers.add(logger); + }); + } + + // Wait for the assets to all download + try { + executor.shutdown(); + + if (executor.awaitTermination(2, TimeUnit.HOURS)) { + executor.shutdownNow(); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } finally { + loggers.forEach(ProgressLoggerHelper::completed); + } + } + + private MinecraftVersionMeta.AssetIndex getAssetIndexMeta() { + MinecraftVersionMeta versionInfo = getExtension().getMinecraftProvider().getVersionInfo(); + return versionInfo.assetIndex(); + } + + private AssetIndex getAssetIndex() throws IOException { + final LoomGradleExtension extension = getExtension(); + final MinecraftProvider minecraftProvider = extension.getMinecraftProvider(); + + MinecraftVersionMeta.AssetIndex assetIndex = getAssetIndexMeta(); + File assetsInfo = new File(getAssetsDirectory().get().getAsFile(), "indexes" + File.separator + assetIndex.fabricId(minecraftProvider.minecraftVersion()) + ".json"); + + getProject().getLogger().info(":downloading asset index"); + + if (getProject().getGradle().getStartParameter().isOffline()) { + if (assetsInfo.exists()) { + // We know it's outdated but can't do anything about it, oh well + getProject().getLogger().warn("Asset index outdated"); + } else { + // We don't know what assets we need, just that we don't have any + throw new GradleException("Asset index not found at " + assetsInfo.getAbsolutePath()); + } + } else { + HashedDownloadUtil.downloadIfInvalid(new URL(assetIndex.url()), assetsInfo, assetIndex.sha1(), getProject().getLogger(), false); + } + + try (FileReader fileReader = new FileReader(assetsInfo)) { + return LoomGradlePlugin.OBJECT_MAPPER.readValue(fileReader, AssetIndex.class); + } + } + + private File getAssetsFile(AssetIndex.Object object, AssetIndex index) { + if (index.mapToResources() || index.virtual()) { + return new File(getLegacyResourcesDirectory().get().getAsFile(), object.path()); + } + + final String filename = "objects" + File.separator + object.hash().substring(0, 2) + File.separator + object.hash(); + return new File(getAssetsDirectory().get().getAsFile(), filename); } } diff --git a/src/main/java/net/fabricmc/loom/task/launch/GenerateDLIConfigTask.java b/src/main/java/net/fabricmc/loom/task/launch/GenerateDLIConfigTask.java index f3259579..144f9fa9 100644 --- a/src/main/java/net/fabricmc/loom/task/launch/GenerateDLIConfigTask.java +++ b/src/main/java/net/fabricmc/loom/task/launch/GenerateDLIConfigTask.java @@ -39,6 +39,7 @@ import org.apache.commons.io.FileUtils; import org.gradle.api.logging.configuration.ConsoleOutput; import org.gradle.api.tasks.TaskAction; +import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionMeta; import net.fabricmc.loom.task.AbstractLoomTask; public abstract class GenerateDLIConfigTask extends AbstractLoomTask { @@ -46,6 +47,13 @@ public abstract class GenerateDLIConfigTask extends AbstractLoomTask { public void run() throws IOException { final String nativesPath = getExtension().getFiles().getNativesDirectory(getProject()).getAbsolutePath(); + final MinecraftVersionMeta versionInfo = getExtension().getMinecraftProvider().getVersionInfo(); + File assetsDirectory = new File(getExtension().getFiles().getUserCache(), "assets"); + + if (versionInfo.assets().equals("legacy")) { + assetsDirectory = new File(assetsDirectory, "/legacy/" + versionInfo.id()); + } + final LaunchConfig launchConfig = new LaunchConfig() .property("fabric.development", "true") .property("fabric.remapClasspathFile", getExtension().getFiles().getRemapClasspathFile().getAbsolutePath()) @@ -58,7 +66,7 @@ public abstract class GenerateDLIConfigTask extends AbstractLoomTask { .argument("client", "--assetIndex") .argument("client", getExtension().getMinecraftProvider().getVersionInfo().assetIndex().fabricId(getExtension().getMinecraftProvider().minecraftVersion())) .argument("client", "--assetsDir") - .argument("client", new File(getExtension().getFiles().getUserCache(), "assets").getAbsolutePath()); + .argument("client", assetsDirectory.getAbsolutePath()); final boolean plainConsole = getProject().getGradle().getStartParameter().getConsoleOutput() == ConsoleOutput.Plain; final boolean ansiSupportedIDE = new File(getProject().getRootDir(), ".vscode").exists() diff --git a/src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java b/src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java index 4f01e7d4..0d231d14 100644 --- a/src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java +++ b/src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java @@ -35,7 +35,6 @@ import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; -import com.google.common.hash.Hashing; import com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.gradle.api.logging.Logger; @@ -92,13 +91,6 @@ public class HashedDownloadUtil { throw e; } - if (!Checksum.equals(to, expectedHash)) { - String actualHash = Files.asByteSource(to).hash(Hashing.sha1()).toString(); - delete(to); - - throw new IOException(String.format("Downloaded file from %s to %s and got unexpected hash of %s expected %s", from, to, actualHash, expectedHash)); - } - saveSha1(to, expectedHash, logger); }