imports/architectury-loom
1
0
Fork 0
mirror of https://github.com/architectury/architectury-loom.git synced 2026-04-01 21:17:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup asset downloading, add support for legacy and pre-1.6 assets (#585)

Browse Source
This commit is contained in:
modmuss50
2022-01-28 20:26:16 +00:00
committed by GitHub
parent 9286ef9163
commit 4b52176499
6 changed files with 199 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetIndex.java
View File

@@ -24,18 +24,38 @@
package net.fabricmc.loom.configuration.providers.minecraft.assets;
import java.util.HashSet;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonProperty;
@SuppressWarnings("unused")
public record AssetIndex(Map<String, AssetObject> objects, boolean virtual) {
public record AssetIndex(Map<String, Entry> objects, boolean virtual, @JsonProperty("map_to_resources") boolean mapToResources) {
public AssetIndex() {
this(new LinkedHashMap<>(), false);
this(new LinkedHashMap<>(), false, false);
}
public Set<AssetObject> getUniqueObjects() {
return new HashSet<>(this.objects.values());
public Collection<Object> getObjects() {
return objects.entrySet().stream().map(Object::new).toList();
}
public record Entry(String hash, long size) {
}
public record Object(String path, String hash, long size) {
private Object(Map.Entry<String, Entry> entry) {
this(entry.getKey(), entry.getValue().hash(), entry.getValue().size());
}
public String name() {
int end = path().lastIndexOf("/") + 1;
if (end > 0) {
return path().substring(end);
}
return path();
}
}
}

29
src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/AssetObject.java
View File

@@ -1,29 +0,0 @@
/*
* This file is part of fabric-loom, licensed under the MIT License (MIT).
*
* Copyright (c) 2016-2021 FabricMC
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.fabricmc.loom.configuration.providers.minecraft.assets;
@SuppressWarnings("unused")
public record AssetObject(String hash, long size) {
}

162
src/main/java/net/fabricmc/loom/configuration/providers/minecraft/assets/MinecraftAssetsProvider.java
View File

@@ -1,162 +0,0 @@
/*
* This file is part of fabric-loom, licensed under the MIT License (MIT).
*
* Copyright (c) 2018-2021 FabricMC
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.fabricmc.loom.configuration.providers.minecraft.assets;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URL;
import java.util.Deque;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import net.fabricmc.loom.LoomGradleExtension;
import net.fabricmc.loom.LoomGradlePlugin;
import net.fabricmc.loom.configuration.providers.minecraft.MinecraftProvider;
import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionMeta;
import net.fabricmc.loom.util.MirrorUtil;
import net.fabricmc.loom.util.HashedDownloadUtil;
import net.fabricmc.loom.util.gradle.ProgressLoggerHelper;
public class MinecraftAssetsProvider {
public static void provide(MinecraftProvider minecraftProvider, Project project) throws IOException {
LoomGradleExtension extension = LoomGradleExtension.get(project);
boolean offline = project.getGradle().getStartParameter().isOffline();
MinecraftVersionMeta versionInfo = minecraftProvider.getVersionInfo();
MinecraftVersionMeta.AssetIndex assetIndex = versionInfo.assetIndex();
// get existing cache files
File assets = new File(extension.getFiles().getUserCache(), "assets");
if (!assets.exists()) {
assets.mkdirs();
}
File assetsInfo = new File(assets, "indexes" + File.separator + assetIndex.fabricId(minecraftProvider.minecraftVersion()) + ".json");
project.getLogger().info(":downloading asset index");
if (offline) {
if (assetsInfo.exists()) {
//We know it's outdated but can't do anything about it, oh well
project.getLogger().warn("Asset index outdated");
} else {
//We don't know what assets we need, just that we don't have any
throw new GradleException("Asset index not found at " + assetsInfo.getAbsolutePath());
}
} else {
HashedDownloadUtil.downloadIfInvalid(new URL(assetIndex.url()), assetsInfo, assetIndex.sha1(), project.getLogger(), false);
}
Deque<ProgressLoggerHelper> loggers = new ConcurrentLinkedDeque<>();
ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)));
AssetIndex index;
try (FileReader fileReader = new FileReader(assetsInfo)) {
index = LoomGradlePlugin.OBJECT_MAPPER.readValue(fileReader, AssetIndex.class);
}
Stopwatch stopwatch = Stopwatch.createStarted();
Map<String, AssetObject> parent = index.objects();
for (Map.Entry<String, AssetObject> entry : parent.entrySet()) {
AssetObject object = entry.getValue();
String sha1 = object.hash();
String filename = "objects" + File.separator + sha1.substring(0, 2) + File.separator + sha1;
File file = new File(assets, filename);
if (offline) {
if (file.exists()) {
project.getLogger().warn("Outdated asset " + entry.getKey());
} else {
throw new GradleException("Asset " + entry.getKey() + " not found at " + file.getAbsolutePath());
}
} else {
executor.execute(() -> {
final String[] assetName = {entry.getKey()};
int end = assetName[0].lastIndexOf("/") + 1;
if (end > 0) {
assetName[0] = assetName[0].substring(end);
}
project.getLogger().debug("validating asset " + assetName[0]);
final ProgressLoggerHelper[] progressLogger = new ProgressLoggerHelper[1];
try {
HashedDownloadUtil.downloadIfInvalid(new URL(MirrorUtil.getResourcesBase(project) + sha1.substring(0, 2) + "/" + sha1), file, sha1, project.getLogger(), true, () -> {
ProgressLoggerHelper logger = loggers.pollFirst();
if (logger == null) {
//Create a new logger if we need one
progressLogger[0] = ProgressLoggerHelper.getProgressFactory(project, MinecraftAssetsProvider.class.getName());
progressLogger[0].start("Downloading assets...", "assets");
} else {
// use a free logger if we can
progressLogger[0] = logger;
}
project.getLogger().debug("downloading asset " + assetName[0]);
progressLogger[0].progress(String.format("%-30.30s", assetName[0]) + " - " + sha1);
});
} catch (IOException e) {
throw new RuntimeException("Failed to download: " + assetName[0], e);
}
if (progressLogger[0] != null) {
//Give this logger back if we used it
loggers.add(progressLogger[0]);
}
});
}
}
project.getLogger().info("Took " + stopwatch.stop() + " to iterate " + parent.size() + " asset index.");
//Wait for the assets to all download
executor.shutdown();
try {
if (executor.awaitTermination(2, TimeUnit.HOURS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
loggers.forEach(ProgressLoggerHelper::completed);
}
}

169
src/main/java/net/fabricmc/loom/task/DownloadAssetsTask.java
View File

@@ -24,20 +24,179 @@
package net.fabricmc.loom.task;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;
import java.util.Deque;
import java.util.Objects;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import javax.inject.Inject;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputDirectory;
import org.gradle.api.tasks.TaskAction;
import net.fabricmc.loom.LoomGradleExtension;
import net.fabricmc.loom.configuration.providers.minecraft.assets.MinecraftAssetsProvider;
import net.fabricmc.loom.LoomGradlePlugin;
import net.fabricmc.loom.configuration.ide.RunConfigSettings;
import net.fabricmc.loom.configuration.providers.minecraft.MinecraftProvider;
import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionMeta;
import net.fabricmc.loom.configuration.providers.minecraft.assets.AssetIndex;
import net.fabricmc.loom.util.HashedDownloadUtil;
import net.fabricmc.loom.util.MirrorUtil;
import net.fabricmc.loom.util.gradle.ProgressLoggerHelper;
public abstract class DownloadAssetsTask extends AbstractLoomTask {
@Input
public abstract Property<String> getAssetsHash();
@OutputDirectory
public abstract RegularFileProperty getAssetsDirectory();
@OutputDirectory
public abstract RegularFileProperty getLegacyResourcesDirectory();
@Inject
public DownloadAssetsTask() {
final MinecraftVersionMeta versionInfo = getExtension().getMinecraftProvider().getVersionInfo();
final File assetsDir = new File(getExtension().getFiles().getUserCache(), "assets");
getAssetsDirectory().set(assetsDir);
getAssetsHash().set(versionInfo.assetIndex().sha1());
if (versionInfo.assets().equals("legacy")) {
getLegacyResourcesDirectory().set(new File(assetsDir, "/legacy/" + versionInfo.id()));
} else {
// pre-1.6 resources
RunConfigSettings client = Objects.requireNonNull(getExtension().getRunConfigs().findByName("client"), "Could not find client run config");
getLegacyResourcesDirectory().set(new File(getProject().getProjectDir(), client.getRunDir() + "/resources"));
}
getAssetsHash().finalizeValueOnRead();
getAssetsDirectory().finalizeValueOnRead();
getLegacyResourcesDirectory().finalizeValueOnRead();
}
public class DownloadAssetsTask extends AbstractLoomTask {
@TaskAction
public void downloadAssets() throws IOException {
Project project = this.getProject();
LoomGradleExtension extension = getExtension();
final Project project = this.getProject();
final File assetsDirectory = getAssetsDirectory().get().getAsFile();
final Deque<ProgressLoggerHelper> loggers = new ConcurrentLinkedDeque<>();
final ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)));
final AssetIndex assetIndex = getAssetIndex();
MinecraftAssetsProvider.provide(extension.getMinecraftProvider(), project);
if (!assetsDirectory.exists()) {
assetsDirectory.mkdirs();
}
if (assetIndex.mapToResources()) {
getLegacyResourcesDirectory().get().getAsFile().mkdirs();
}
for (AssetIndex.Object object : assetIndex.getObjects()) {
final String path = object.path();
final String sha1 = object.hash();
final File file = getAssetsFile(object, assetIndex);
if (getProject().getGradle().getStartParameter().isOffline()) {
if (!file.exists()) {
throw new GradleException("Asset " + path + " not found at " + file.getAbsolutePath());
}
continue;
}
final Supplier<ProgressLoggerHelper> getOrCreateLogger = () -> {
ProgressLoggerHelper logger = loggers.pollFirst();
if (logger == null) {
// No logger available, create a new one
logger = ProgressLoggerHelper.getProgressFactory(project, DownloadAssetsTask.class.getName());
logger.start("Downloading assets...", "assets");
}
return logger;
};
executor.execute(() -> {
final ProgressLoggerHelper logger = getOrCreateLogger.get();
try {
HashedDownloadUtil.downloadIfInvalid(new URL(MirrorUtil.getResourcesBase(project) + sha1.substring(0, 2) + "/" + sha1), file, sha1, project.getLogger(), true, () -> {
project.getLogger().debug("downloading asset " + object.name());
logger.progress(String.format("%-30.30s", object.name()) + " - " + sha1);
});
} catch (IOException e) {
throw new UncheckedIOException("Failed to download: " + object.name(), e);
}
// Give this logger back
loggers.add(logger);
});
}
// Wait for the assets to all download
try {
executor.shutdown();
if (executor.awaitTermination(2, TimeUnit.HOURS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
} finally {
loggers.forEach(ProgressLoggerHelper::completed);
}
}
private MinecraftVersionMeta.AssetIndex getAssetIndexMeta() {
MinecraftVersionMeta versionInfo = getExtension().getMinecraftProvider().getVersionInfo();
return versionInfo.assetIndex();
}
private AssetIndex getAssetIndex() throws IOException {
final LoomGradleExtension extension = getExtension();
final MinecraftProvider minecraftProvider = extension.getMinecraftProvider();
MinecraftVersionMeta.AssetIndex assetIndex = getAssetIndexMeta();
File assetsInfo = new File(getAssetsDirectory().get().getAsFile(), "indexes" + File.separator + assetIndex.fabricId(minecraftProvider.minecraftVersion()) + ".json");
getProject().getLogger().info(":downloading asset index");
if (getProject().getGradle().getStartParameter().isOffline()) {
if (assetsInfo.exists()) {
// We know it's outdated but can't do anything about it, oh well
getProject().getLogger().warn("Asset index outdated");
} else {
// We don't know what assets we need, just that we don't have any
throw new GradleException("Asset index not found at " + assetsInfo.getAbsolutePath());
}
} else {
HashedDownloadUtil.downloadIfInvalid(new URL(assetIndex.url()), assetsInfo, assetIndex.sha1(), getProject().getLogger(), false);
}
try (FileReader fileReader = new FileReader(assetsInfo)) {
return LoomGradlePlugin.OBJECT_MAPPER.readValue(fileReader, AssetIndex.class);
}
}
private File getAssetsFile(AssetIndex.Object object, AssetIndex index) {
if (index.mapToResources() || index.virtual()) {
return new File(getLegacyResourcesDirectory().get().getAsFile(), object.path());
}
final String filename = "objects" + File.separator + object.hash().substring(0, 2) + File.separator + object.hash();
return new File(getAssetsDirectory().get().getAsFile(), filename);
}
}

10
src/main/java/net/fabricmc/loom/task/launch/GenerateDLIConfigTask.java
View File

@@ -39,6 +39,7 @@ import org.apache.commons.io.FileUtils;
import org.gradle.api.logging.configuration.ConsoleOutput;
import org.gradle.api.tasks.TaskAction;
import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionMeta;
import net.fabricmc.loom.task.AbstractLoomTask;
public abstract class GenerateDLIConfigTask extends AbstractLoomTask {
@@ -46,6 +47,13 @@ public abstract class GenerateDLIConfigTask extends AbstractLoomTask {
public void run() throws IOException {
final String nativesPath = getExtension().getFiles().getNativesDirectory(getProject()).getAbsolutePath();
final MinecraftVersionMeta versionInfo = getExtension().getMinecraftProvider().getVersionInfo();
File assetsDirectory = new File(getExtension().getFiles().getUserCache(), "assets");
if (versionInfo.assets().equals("legacy")) {
assetsDirectory = new File(assetsDirectory, "/legacy/" + versionInfo.id());
}
final LaunchConfig launchConfig = new LaunchConfig()
.property("fabric.development", "true")
.property("fabric.remapClasspathFile", getExtension().getFiles().getRemapClasspathFile().getAbsolutePath())
@@ -58,7 +66,7 @@ public abstract class GenerateDLIConfigTask extends AbstractLoomTask {
.argument("client", "--assetIndex")
.argument("client", getExtension().getMinecraftProvider().getVersionInfo().assetIndex().fabricId(getExtension().getMinecraftProvider().minecraftVersion()))
.argument("client", "--assetsDir")
.argument("client", new File(getExtension().getFiles().getUserCache(), "assets").getAbsolutePath());
.argument("client", assetsDirectory.getAbsolutePath());
final boolean plainConsole = getProject().getGradle().getStartParameter().getConsoleOutput() == ConsoleOutput.Plain;
final boolean ansiSupportedIDE = new File(getProject().getRootDir(), ".vscode").exists()

8
src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java
View File

@@ -35,7 +35,6 @@ import java.util.zip.GZIPInputStream;
import javax.annotation.Nullable;
import com.google.common.hash.Hashing;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.gradle.api.logging.Logger;
@@ -92,13 +91,6 @@ public class HashedDownloadUtil {
throw e;
}
if (!Checksum.equals(to, expectedHash)) {
String actualHash = Files.asByteSource(to).hash(Hashing.sha1()).toString();
delete(to);
throw new IOException(String.format("Downloaded file from %s to %s and got unexpected hash of %s expected %s", from, to, actualHash, expectedHash));
}
saveSha1(to, expectedHash, logger);
}